9#include <similie/tensor/owning_tensor.hpp>
11#include "minimize_strong_formulation_residual.hpp"
16 std::shared_ptr<gko::matrix::Csr<double, gko::int32>>
matrix;
17 Kokkos::View<double**>
rhs;
22namespace similie::solvers::detail {
27template <std::
size_t Capacity>
28KOKKOS_FUNCTION
auto make_row_stencil()
30 std::array<double, Capacity> storage {};
31 ddc::DiscreteDomain<StencilColumn>
const
32 domain(ddc::DiscreteElement<StencilColumn>(0),
33 ddc::DiscreteVector<StencilColumn>(Capacity));
36 ddc::DiscreteDomain<StencilColumn>,
38 Kokkos::DefaultExecutionSpace::memory_space>
39 tensor(ddc::ChunkSpan<
41 ddc::DiscreteDomain<StencilColumn>,
43 Kokkos::DefaultExecutionSpace::memory_space>(storage.data(), domain));
47template <
class Weights,
class Indices>
48KOKKOS_FUNCTION
bool add_stencil_coefficient(
49 Weights
const& weights,
58 for (; i < count; ++i)
59 if (indices[i] == column)
62 if (count == weights.domain().size())
64 indices[count++] = column;
66 weights.mem(ddc::DiscreteElement<StencilColumn>(i)) += weight;
72template <std::
size_t Capacity>
75 static_assert(Capacity > 0);
76 std::size_t column_count;
77 Kokkos::View<std::size_t**, Kokkos::LayoutRight> columns;
78 Kokkos::View<double**, Kokkos::LayoutRight> weights;
79 Kokkos::View<std::size_t*> counts;
80 Kokkos::View<double*> constants, lower, upper;
81 Kokkos::View<int> errors;
83 explicit RowAssembly(std::size_t size,
bool matrix =
true, std::size_t input_size = 0)
84 : column_count(input_size == 0 ? size : input_size)
85 , columns(
"dec_columns", matrix ? size : 0, Capacity)
86 , weights(
"dec_weights", matrix ? size : 0, Capacity)
87 , counts(
"dec_counts", matrix ? size : 0)
88 , constants(
"dec_constants", size)
89 , lower(
"dirichlet_lower", size)
90 , upper(
"dirichlet_upper", size)
91 , errors(
"dec_errors")
93 if (size == 0 || size > std::size_t(std::numeric_limits<int>::max()) / Capacity
94 || column_count > std::size_t(std::numeric_limits<int>::max()))
95 throw std::invalid_argument(
"differential matrix exceeds 32-bit CSR capacity");
96 Kokkos::deep_copy(lower, std::numeric_limits<double>::infinity());
97 Kokkos::deep_copy(upper, -std::numeric_limits<double>::infinity());
100 KOKKOS_FUNCTION
void fail(
int code)
const
102 Kokkos::atomic_fetch_or(&errors(), code);
105 KOKKOS_FUNCTION
auto coefficients(std::size_t row)
const
107 ddc::DiscreteDomain<StencilColumn>
const
108 domain(ddc::DiscreteElement<StencilColumn>(0),
109 ddc::DiscreteVector<StencilColumn>(Capacity));
112 ddc::DiscreteDomain<StencilColumn>,
113 Kokkos::layout_right,
114 Kokkos::DefaultExecutionSpace::memory_space>(
117 ddc::DiscreteDomain<StencilColumn>,
118 Kokkos::layout_right,
119 Kokkos::DefaultExecutionSpace::memory_space>(&weights(row, 0), domain));
122 KOKKOS_FUNCTION
void add(std::size_t row, std::size_t column,
double weight)
const
124 if (column >= column_count) {
128 if (!add_stencil_coefficient(
137 KOKKOS_FUNCTION
void prescribe(std::size_t index,
double value)
const
139 if (index >= constants.extent(0) || index >= column_count || !Kokkos::isfinite(value)) {
143 Kokkos::atomic_min(&lower(index), value);
144 Kokkos::atomic_max(&upper(index), value);
150 Kokkos::deep_copy(error, errors);
152 throw std::runtime_error(
"differential stencil exceeds its column capacity");
154 throw std::runtime_error(
"incompatible or invalid Dirichlet constraints");
156 throw std::runtime_error(
"invalid or empty differential row");
159 Kokkos::View<double**> finalize_rhs(
160 Kokkos::DefaultExecutionSpace
const& exec,
161 Kokkos::View<double*>
const& scales)
const
164 RowAssembly
const data = *
this;
165 Kokkos::View<double**> rhs(
"dec_rhs", constants.extent(0), 1);
166 Kokkos::parallel_for(
168 Kokkos::RangePolicy(exec, 0, constants.extent(0)),
169 KOKKOS_LAMBDA(std::size_t row) {
170 if (Kokkos::isfinite(data.lower(row))) {
171 if (Kokkos::abs(data.upper(row) - data.lower(row))
172 > 1.e-12 * (1.0 + Kokkos::abs(data.lower(row))))
174 data.constants(row) = -data.lower(row);
176 if (!Kokkos::isfinite(data.constants(row)))
178 rhs(row, 0) = -data.constants(row) / scales(row);
180 exec.fence(
"scalar DEC rhs ready");
185 AssembledLinearSystem finalize(
186 Kokkos::DefaultExecutionSpace
const& exec,
188 bool allow_empty_rows =
false)
const
191 RowAssembly
const data = *
this;
192 Kokkos::View<int*> offsets(
"dec_offsets", constants.extent(0) + 1);
193 Kokkos::View<double*> scales(
"dec_row_scales", constants.extent(0));
194 Kokkos::parallel_for(
196 Kokkos::RangePolicy(exec, 0, constants.extent(0)),
197 KOKKOS_LAMBDA(std::size_t row) {
198 if (Kokkos::isfinite(data.lower(row))) {
199 data.counts(row) = 1;
200 data.columns(row, 0) = row;
201 data.weights(row, 0) = 1.0;
204 std::size_t count = 0;
205 for (std::size_t i = 0; i < data.counts(row); ++i) {
206 double const weight = data.weights(row, i);
207 if (!Kokkos::isfinite(weight))
210 data.columns(row, count) = data.columns(row, i);
211 data.weights(row, count++) = weight;
212 scale = Kokkos::max(scale, Kokkos::abs(weight));
215 data.counts(row) = count;
216 if (!(scale > 0.0)) {
217 if (allow_empty_rows)
222 scales(row) = normalize ? scale : 1.0;
223 for (std::size_t i = 0; i < count; ++i)
224 data.weights(row, i) /= scales(row);
227 Kokkos::parallel_scan(
229 Kokkos::RangePolicy(exec, 0, constants.extent(0) + 1),
230 KOKKOS_LAMBDA(std::size_t row,
int& sum,
bool final) {
233 if (row < data.constants.extent(0))
234 sum +=
static_cast<int>(data.counts(row));
238 Kokkos::View<int*> csr_columns(
"dec_csr_columns", total);
239 Kokkos::View<double*> values(
"dec_values", total);
240 Kokkos::parallel_for(
242 Kokkos::RangePolicy(exec, 0, constants.extent(0)),
243 KOKKOS_LAMBDA(std::size_t row) {
244 for (std::size_t i = 0; i < data.counts(row); ++i) {
245 csr_columns(offsets(row) + i) =
static_cast<int>(data.columns(row, i));
246 values(offsets(row) + i) = data.weights(row, i);
249 Kokkos::View<double**>
const rhs = finalize_rhs(exec, scales);
250 auto const executor = gko::ext::kokkos::create_executor(exec);
251 gko::array<double> matrix_values(executor, values.extent(0));
252 gko::array<gko::int32> matrix_columns(executor, csr_columns.extent(0)),
253 matrix_offsets(executor, offsets.extent(0));
258 matrix_values.get_data());
261 csr_columns.extent(0),
263 matrix_columns.get_data());
268 matrix_offsets.get_data());
269 std::shared_ptr<gko::matrix::Csr<double, gko::int32>> matrix
270 = gko::matrix::Csr<double, gko::int32>::
272 gko::dim<2>(constants.extent(0), column_count),
273 std::move(matrix_values),
274 std::move(matrix_columns),
275 std::move(matrix_offsets));
276 matrix->sort_by_column_index();
277 return {matrix, rhs, scales};
OwningTensor(TensorType, Storage &&) -> OwningTensor< TensorType, Storage >
Kokkos::View< double * > row_scales
std::shared_ptr< gko::matrix::Csr< double, gko::int32 > > matrix
Kokkos::View< double ** > rhs