71 switch (preconditioner) {
79 return "SymmetricGaussSeidel";
83 return "ChebyshevJacobi";
96 if (name ==
"Identity" || name ==
"identity") {
99 if (name ==
"Jacobi" || name ==
"jacobi") {
102 if (name ==
"SpdIsai" || name ==
"spd-isai" || name ==
"spd_isai") {
105 if (name ==
"SymmetricGaussSeidel" || name ==
"symmetric-gauss-seidel"
106 || name ==
"symmetric_gauss_seidel") {
109 if (name ==
"Ssor" || name ==
"SSOR" || name ==
"ssor") {
112 if (name ==
"ChebyshevJacobi" || name ==
"chebyshev-jacobi" || name ==
"chebyshev_jacobi") {
115 if (name ==
"IrJacobi" || name ==
"ir-jacobi" || name ==
"ir_jacobi") {
118 if (name ==
"GeneralIsai" || name ==
"general-isai" || name ==
"general_isai"
122 if (name ==
"Multigrid" || name ==
"multigrid" || name ==
"amg") {
125 throw std::runtime_error(
"unknown strong-formulation preconditioner: " + std::string(name));
161inline unsigned int solver_progress_stride()
163 char const*
const value = std::getenv(
"SIMILIE_SOLVER_PROGRESS_STRIDE");
164 if (value ==
nullptr || value[0] ==
'\0') {
168 char* parse_end =
nullptr;
169 unsigned long const parsed = std::strtoul(value, &parse_end, 10);
170 if (parse_end == value) {
176 if (parsed > std::numeric_limits<unsigned int>::max()) {
177 return std::numeric_limits<unsigned int>::max();
179 return static_cast<unsigned int>(parsed);
182inline bool solver_progress_enabled()
184 return solver_progress_stride() != 0U;
187inline bool env_flag_enabled(
char const* name)
189 char const*
const value = std::getenv(name);
190 return value !=
nullptr && value[0] !=
'\0' && value[0] !=
'0';
193inline bool env_value_equals(
char const* name,
char const* expected)
195 char const*
const value = std::getenv(name);
196 return value !=
nullptr && std::string_view(value) == expected;
199inline double env_double_or(
char const* name,
double default_value)
201 char const*
const value = std::getenv(name);
202 if (value ==
nullptr || value[0] ==
'\0') {
203 return default_value;
205 char* parse_end =
nullptr;
206 double const parsed = std::strtod(value, &parse_end);
207 if (parse_end == value) {
208 return default_value;
213inline int env_int_or(
char const* name,
int default_value)
215 char const*
const value = std::getenv(name);
216 if (value ==
nullptr || value[0] ==
'\0') {
217 return default_value;
219 char* parse_end =
nullptr;
220 long const parsed = std::strtol(value, &parse_end, 10);
221 if (parse_end == value) {
222 return default_value;
224 if (parsed > std::numeric_limits<int>::max()) {
225 return std::numeric_limits<int>::max();
227 if (parsed < std::numeric_limits<int>::min()) {
228 return std::numeric_limits<int>::min();
230 return static_cast<int>(parsed);
233template <
class MatrixData>
234void log_matrix_diagnostics(MatrixData
const& matrix_data)
236 std::size_t
const size = matrix_data.size[0];
237 std::vector<double> diagonal(size, 0.0);
238 std::size_t negative_diagonal_count = 0;
239 std::size_t zero_diagonal_count = 0;
240 double min_diagonal = std::numeric_limits<double>::infinity();
241 double max_diagonal = -std::numeric_limits<double>::infinity();
242 std::vector<double> off_diagonal_abs_row_sum(size, 0.0);
243 for (
auto const& entry : matrix_data.nonzeros) {
244 if (entry.row == entry.column) {
245 diagonal[
static_cast<std::size_t
>(entry.row)] += entry.value;
247 off_diagonal_abs_row_sum[
static_cast<std::size_t
>(entry.row)] += std::abs(entry.value);
250 double jacobi_gershgorin_lower = std::numeric_limits<double>::infinity();
251 double jacobi_gershgorin_upper = -std::numeric_limits<double>::infinity();
252 double jacobi_max_abs_off_diagonal_row_sum = 0.0;
253 for (
double const value : diagonal) {
255 ++negative_diagonal_count;
258 ++zero_diagonal_count;
260 min_diagonal = std::min(min_diagonal, value);
261 max_diagonal = std::max(max_diagonal, value);
263 for (std::size_t row = 0; row < size; ++row) {
264 if (diagonal[row] == 0.0) {
267 double const scaled_radius = off_diagonal_abs_row_sum[row] / std::abs(diagonal[row]);
268 jacobi_max_abs_off_diagonal_row_sum
269 = std::max(jacobi_max_abs_off_diagonal_row_sum, scaled_radius);
270 jacobi_gershgorin_lower = std::min(jacobi_gershgorin_lower, 1.0 - scaled_radius);
271 jacobi_gershgorin_upper = std::max(jacobi_gershgorin_upper, 1.0 + scaled_radius);
274 double max_abs_asymmetry = 0.0;
275 std::size_t asymmetric_entry_count = 0;
276 bool const symmetry_diagnostics_skipped
277 = env_flag_enabled(
"SIMILIE_SKIP_MATRIX_SYMMETRY_DIAGNOSTICS");
278 if (!symmetry_diagnostics_skipped) {
279 auto matrix_entries = std::unordered_map<std::uint64_t, double>();
280 matrix_entries.reserve(matrix_data.nonzeros.size());
281 auto const entry_key = [size](std::size_t row, std::size_t column) {
282 return static_cast<std::uint64_t
>(row) *
static_cast<std::uint64_t
>(size)
283 +
static_cast<std::uint64_t
>(column);
285 for (
auto const& entry : matrix_data.nonzeros) {
286 matrix_entries[entry_key(
287 static_cast<std::size_t
>(entry.row),
288 static_cast<std::size_t
>(entry.column))]
291 for (
auto const& [key, value] : matrix_entries) {
292 std::size_t
const row
293 =
static_cast<std::size_t
>(key /
static_cast<std::uint64_t
>(size));
294 std::size_t
const column
295 =
static_cast<std::size_t
>(key %
static_cast<std::uint64_t
>(size));
296 double transposed_value = 0.0;
297 if (
auto const transposed = matrix_entries.find(entry_key(column, row));
298 transposed != matrix_entries.end()) {
299 transposed_value = transposed->second;
301 double const asymmetry = std::abs(value - transposed_value);
303 > 1.0e-10 * std::max({1.0, std::abs(value), std::abs(transposed_value)})) {
304 ++asymmetric_entry_count;
306 max_abs_asymmetry = std::max(max_abs_asymmetry, asymmetry);
310 std::cout <<
"SimiLie matrix diagnostics: size=" << size
311 <<
" nonzeros=" << matrix_data.nonzeros.size() <<
" diagonal_min=" << min_diagonal
312 <<
" diagonal_max=" << max_diagonal
313 <<
" diagonal_negative_count=" << negative_diagonal_count
314 <<
" diagonal_zero_count=" << zero_diagonal_count
315 <<
" jacobi_gershgorin_lower=" << jacobi_gershgorin_lower
316 <<
" jacobi_gershgorin_upper=" << jacobi_gershgorin_upper
317 <<
" jacobi_max_abs_off_diagonal_row_sum=" << jacobi_max_abs_off_diagonal_row_sum
318 <<
" symmetry_diagnostics_skipped=" << symmetry_diagnostics_skipped
319 <<
" asymmetric_entry_count=" << asymmetric_entry_count
320 <<
" max_abs_asymmetry=" << max_abs_asymmetry <<
'\n';
323class SolverProgressLogger final :
public gko::log::Logger
326 SolverProgressLogger(
327 std::shared_ptr<gko::Executor const> master_executor,
328 double initial_residual_l2,
330 : Logger(gko::log::Logger::iteration_complete_mask)
331 , m_master_executor(std::move(master_executor))
332 , m_initial_residual_l2(initial_residual_l2)
338 using gko::log::Logger::on_iteration_complete;
340 void on_iteration_complete(
344 gko::size_type
const& num_iterations,
345 gko::LinOp
const* residual,
346 gko::LinOp
const* residual_norm,
348 gko::array<gko::stopping_status>
const*,
351 if (m_stride == 0U || num_iterations % m_stride != 0U) {
354 std::optional<double> residual_l2 = residual_norm_value(residual_norm);
355 if (!residual_l2.has_value()) {
356 residual_l2 = residual_vector_l2(residual);
358 if (!residual_l2.has_value()) {
361 double const relative_residual
362 = m_initial_residual_l2 == 0.0 ? 0.0 : *residual_l2 / m_initial_residual_l2;
363 std::cout <<
"SimiLie solver progress: iteration=" << num_iterations
364 <<
" residual_l2=" << *residual_l2 <<
" relative_residual=" << relative_residual
369 std::optional<double> residual_norm_value(gko::LinOp
const* residual_norm)
const
371 auto const* residual_norm_dense
372 =
dynamic_cast<gko::matrix::Dense<double> const*
>(residual_norm);
373 if (residual_norm_dense ==
nullptr) {
376 auto host_dense = gko::matrix::Dense<
377 double>::create(m_master_executor, residual_norm_dense->get_size());
378 residual_norm_dense->convert_to(host_dense.get());
379 return host_dense->at(0, 0);
382 std::optional<double> residual_vector_l2(gko::LinOp
const* residual)
const
384 auto const* residual_dense =
dynamic_cast<gko::matrix::Dense<double> const*
>(residual);
385 if (residual_dense ==
nullptr) {
389 = gko::matrix::Dense<double>::create(m_master_executor, residual_dense->get_size());
390 residual_dense->convert_to(host_dense.get());
391 double squared_norm = 0.0;
392 for (gko::size_type row = 0; row < host_dense->get_size()[0]; ++row) {
393 for (gko::size_type column = 0; column < host_dense->get_size()[1]; ++column) {
394 double const value = host_dense->at(row, column);
395 squared_norm += value * value;
398 return std::sqrt(squared_norm);
401 std::shared_ptr<gko::Executor const> m_master_executor;
402 double m_initial_residual_l2;
403 unsigned int m_stride;
406inline void log_nonlinear_progress(
407 unsigned int iteration,
408 StrongFormulationSolverDiagnostics
const& diagnostics)
410 if (!solver_progress_enabled()) {
413 std::cout <<
"SimiLie nonlinear progress: iteration=" << iteration
414 <<
" residual_l2=" << diagnostics.final_residual_l2
415 <<
" relative_residual=" << diagnostics.final_relative_residual << std::endl;
418template <
class ExecSpace,
class OperatorModel,
class =
void>
419struct MatrixFreeWorkspaceTraits
421 static constexpr bool enabled =
false;
427template <
class ExecSpace,
class OperatorModel>
428struct MatrixFreeWorkspaceTraits<
431 std::void_t<decltype(std::declval<OperatorModel const&>().create_matrix_free_workspace(
432 std::declval<ExecSpace>()))>>
434 static constexpr bool enabled =
true;
435 using type =
decltype(std::declval<OperatorModel const&>().create_matrix_free_workspace(
436 std::declval<ExecSpace>()));
439template <
class OperatorModel>
440bool uses_precomputed_matrix_free_stencils(OperatorModel
const& operator_model)
442 if constexpr (
requires(OperatorModel
const& model) { model.has_precomputed_stencils(); }) {
443 return operator_model.has_precomputed_stencils();
448template <
class ExecSpace,
class ViewType1,
class ViewType2>
449double dot(ExecSpace
const& exec_space, ViewType1
const& lhs, ViewType2
const& rhs)
452 Kokkos::parallel_reduce(
454 Kokkos::MDRangePolicy<
456 Kokkos::Rank<2>>(exec_space, {0, 0}, {lhs.extent(0), lhs.extent(1)}),
457 KOKKOS_LAMBDA(std::size_t row, std::size_t column,
double& local_sum) {
458 local_sum += lhs(row, column) * rhs(row, column);
465template <
class ExecSpace,
class ViewType>
466double residual_norm_l2(ExecSpace
const& exec_space, ViewType
const& residual)
468 return std::sqrt(dot(exec_space, residual, residual));
471template <
class ExecSpace,
class ViewType>
472void fill(ExecSpace
const& exec_space, ViewType
const& view,
double value)
474 Kokkos::parallel_for(
476 Kokkos::MDRangePolicy<
478 Kokkos::Rank<2>>(exec_space, {0, 0}, {view.extent(0), view.extent(1)}),
479 KOKKOS_LAMBDA(std::size_t row, std::size_t column) { view(row, column) = value; });
483template <
class ExecSpace,
class DestinationView,
class SourceView>
484void copy(ExecSpace
const& exec_space, DestinationView
const& destination, SourceView
const& source)
486 Kokkos::parallel_for(
488 Kokkos::MDRangePolicy<
491 2>>(exec_space, {0, 0}, {destination.extent(0), destination.extent(1)}),
492 KOKKOS_LAMBDA(std::size_t row, std::size_t column) {
493 destination(row, column) = source(row, column);
498template <
class ExecSpace,
class ViewType1,
class ViewType2,
class ViewType3>
500 ExecSpace
const& exec_space,
501 ViewType1
const& destination,
507 Kokkos::parallel_for(
509 Kokkos::MDRangePolicy<
512 2>>(exec_space, {0, 0}, {destination.extent(0), destination.extent(1)}),
513 KOKKOS_LAMBDA(std::size_t row, std::size_t column) {
514 destination(row, column) = alpha * x(row, column) + beta * y(row, column);
519template <
class ExecSpace,
class ViewType1,
class ViewType2>
521 ExecSpace
const& exec_space,
522 ViewType1
const& destination,
524 ViewType2
const& source)
526 Kokkos::parallel_for(
527 "similie_axpy_inplace",
528 Kokkos::MDRangePolicy<
531 2>>(exec_space, {0, 0}, {destination.extent(0), destination.extent(1)}),
532 KOKKOS_LAMBDA(std::size_t row, std::size_t column) {
533 destination(row, column) += alpha * source(row, column);
538template <
class KokkosViewType>
541 std::shared_ptr<gko::matrix::Dense<typename KokkosViewType::non_const_value_type>> dense;
542 std::unique_ptr<Kokkos::View<
543 typename KokkosViewType::non_const_value_type**,
545 typename KokkosViewType::memory_space>>
549template <
class KokkosViewType>
550auto to_gko_dense(std::shared_ptr<gko::Executor const>
const& gko_exec, KokkosViewType
const& view)
552 static_assert(Kokkos::is_view_v<KokkosViewType> && KokkosViewType::rank == 2);
553 using value_type =
typename KokkosViewType::traits::value_type;
554 using non_const_value_type =
typename KokkosViewType::non_const_value_type;
555 using owning_view_type = Kokkos::View<
556 non_const_value_type**,
558 typename KokkosViewType::memory_space>;
560 GkoDenseHandle<KokkosViewType> handle;
561 if (view.stride(1) == 1) {
562 handle.dense = gko::matrix::Dense<value_type>::
564 gko::dim<2>(view.extent(0), view.extent(1)),
565 gko::array<value_type>::view(gko_exec, view.span(), view.data()),
570 handle.owned_view = std::make_unique<
571 owning_view_type>(
"similie_gko_dense_bridge", view.extent(0), view.extent(1));
572 Kokkos::deep_copy(*handle.owned_view, view);
573 handle.dense = gko::matrix::Dense<value_type>::
575 gko::dim<2>(handle.owned_view->extent(0), handle.owned_view->extent(1)),
576 gko::array<value_type>::
577 view(gko_exec, handle.owned_view->span(), handle.owned_view->data()),
578 handle.owned_view->stride(0));
582template <
class DestinationView,
class KokkosViewType>
583void copy_back_from_gko_dense_bridge(
584 DestinationView
const& destination,
585 GkoDenseHandle<KokkosViewType>
const& handle)
587 if (handle.owned_view) {
588 Kokkos::deep_copy(destination, *handle.owned_view);
592inline std::shared_ptr<gko::matrix::Csr<double, gko::int32>> csr_from_matrix_data(
593 std::shared_ptr<gko::Executor const>
const& gko_exec,
594 gko::matrix_data<double, gko::int32>
const& matrix_data)
596 using matrix_type = gko::matrix::Csr<double, gko::int32>;
597 std::vector<double> values(matrix_data.nonzeros.size());
598 std::vector<gko::int32> columns(matrix_data.nonzeros.size());
599 std::vector<gko::int32> row_ptrs(matrix_data.size[0] + 1, 0);
600 for (
auto const& nonzero : matrix_data.nonzeros) {
601 ++row_ptrs[
static_cast<std::size_t
>(nonzero.row) + 1];
603 for (std::size_t row = 0; row < matrix_data.size[0]; ++row) {
604 row_ptrs[row + 1] += row_ptrs[row];
607 auto next_offsets = row_ptrs;
608 auto const next_offsets_view = Kokkos::
609 View<gko::int32*, Kokkos::HostSpace>(next_offsets.data(), next_offsets.size());
610 auto const nonzeros_view = Kokkos::View<
611 gko::matrix_data<double, gko::int32>::nonzero_type
const*,
612 Kokkos::HostSpace>(matrix_data.nonzeros.data(), matrix_data.nonzeros.size());
613 auto const columns_view
614 = Kokkos::View<gko::int32*, Kokkos::HostSpace>(columns.data(), columns.size());
615 auto const values_view = Kokkos::View<double*, Kokkos::HostSpace>(values.data(), values.size());
616 Kokkos::parallel_for(
617 "similie_pack_ginkgo_csr_arrays",
618 Kokkos::RangePolicy<Kokkos::DefaultHostExecutionSpace>(
619 Kokkos::DefaultHostExecutionSpace(),
621 static_cast<Kokkos::DefaultHostExecutionSpace::size_type
>(
622 matrix_data.nonzeros.size())),
623 KOKKOS_LAMBDA(std::size_t nonzero_index) {
624 auto const nonzero = nonzeros_view(nonzero_index);
625 gko::int32
const output_index
626 = Kokkos::atomic_fetch_add(&next_offsets_view(nonzero.row), 1);
627 columns_view(
static_cast<std::size_t
>(output_index)) = nonzero.column;
628 values_view(
static_cast<std::size_t
>(output_index)) = nonzero.value;
630 Kokkos::DefaultHostExecutionSpace().fence();
632 return std::shared_ptr<matrix_type>(
636 gko::array<double>(gko_exec, values.begin(), values.end()),
637 gko::array<gko::int32>(gko_exec, columns.begin(), columns.end()),
638 gko::array<gko::int32>(gko_exec, row_ptrs.begin(), row_ptrs.end()))
642template <
class OperatorModel>
643std::shared_ptr<gko::matrix::Csr<double, gko::int32>> build_matrix(
644 std::shared_ptr<gko::Executor const>
const& gko_exec,
645 OperatorModel
const& operator_model)
647 if constexpr (
requires { assemble_matrix_data(operator_model); }) {
648 auto matrix_data = assemble_matrix_data(operator_model);
649 matrix_data.sum_duplicates();
650 if (env_flag_enabled(
"SIMILIE_MATRIX_DIAGNOSTICS"))
651 log_matrix_diagnostics(matrix_data);
652 return csr_from_matrix_data(gko_exec, matrix_data);
654 throw std::invalid_argument(
655 "This operator supports only matrix-free solves with identity preconditioning");
659template <
class OperatorModel,
class StateView>
660std::shared_ptr<gko::matrix::Csr<double, gko::int32>> build_matrix(
661 std::shared_ptr<gko::Executor const>
const& gko_exec,
662 OperatorModel
const& operator_model,
663 StateView
const& state)
665 auto matrix_data = assemble_matrix_data(operator_model, state);
666 matrix_data.sum_duplicates();
667 if (env_flag_enabled(
"SIMILIE_MATRIX_DIAGNOSTICS")) {
668 log_matrix_diagnostics(matrix_data);
670 return csr_from_matrix_data(gko_exec, matrix_data);
673template <
class ExecSpace,
class OperatorModel>
676 ,
public gko::EnableCloneable<MatrixFreeLinOp<ExecSpace, OperatorModel>>
678 using value_type = double;
679 using dense_type = gko::matrix::Dense<value_type>;
680 using memory_space =
typename ExecSpace::memory_space;
681 using base_type = gko::LinOp;
682 using workspace_traits = MatrixFreeWorkspaceTraits<ExecSpace, OperatorModel>;
683 using workspace_type =
typename workspace_traits::type;
685 ExecSpace m_exec_space;
686 std::shared_ptr<OperatorModel const> m_operator_model;
687 mutable std::shared_ptr<workspace_type> m_workspace;
688 mutable Kokkos::View<double**, Kokkos::LayoutRight, memory_space> m_advanced_applied;
689 mutable std::size_t m_apply_count = 0;
690 mutable std::size_t m_advanced_apply_count = 0;
691 mutable double m_apply_duration = 0.0;
692 mutable double m_advanced_apply_duration = 0.0;
695 explicit MatrixFreeLinOp(std::shared_ptr<gko::Executor const> exec)
696 : base_type(std::move(exec))
703 std::shared_ptr<gko::Executor const> exec,
704 ExecSpace
const& exec_space,
705 std::shared_ptr<OperatorModel const> operator_model)
706 : base_type(exec, gko::dim<2>(operator_model->size(), operator_model->size()))
707 , m_exec_space(exec_space)
708 , m_operator_model(std::move(operator_model))
709 , m_advanced_applied(
710 "similie_matrix_free_linop_advanced_workspace",
711 m_operator_model->size(),
714 if constexpr (workspace_traits::enabled) {
715 if (!uses_precomputed_matrix_free_stencils(*m_operator_model)) {
716 m_workspace = std::make_shared<workspace_type>(
717 m_operator_model->create_matrix_free_workspace(m_exec_space));
723 void apply_impl(gko::LinOp
const* b, gko::LinOp* x)
const override
725 log_apply_start(
"simple", m_apply_count + 1);
726 auto const apply_start = std::chrono::steady_clock::now();
727 auto const* b_dense =
dynamic_cast<dense_type const*
>(b);
728 auto* x_dense =
dynamic_cast<dense_type*
>(x);
729 if (b_dense ==
nullptr || x_dense ==
nullptr) {
730 throw std::invalid_argument(
"MatrixFreeLinOp expects dense inputs and outputs");
732 auto b_view = gko::ext::kokkos::map_data<memory_space>(*b_dense);
733 auto x_view = gko::ext::kokkos::map_data<memory_space>(*x_dense);
734 if constexpr (workspace_traits::enabled) {
735 if constexpr (
requires(
736 OperatorModel
const& model,
738 decltype(b_view) input,
739 decltype(x_view) output,
740 workspace_type& workspace) {
741 model.apply(ex, input, output, workspace);
743 if (uses_precomputed_matrix_free_stencils(*m_operator_model)) {
744 m_operator_model->apply(m_exec_space, b_view, x_view);
746 if (m_workspace ==
nullptr) {
747 m_workspace = std::make_shared<workspace_type>(
748 m_operator_model->create_matrix_free_workspace(m_exec_space));
750 m_operator_model->apply(m_exec_space, b_view, x_view, *m_workspace);
753 m_operator_model->apply(m_exec_space, b_view, x_view);
756 m_operator_model->apply(m_exec_space, b_view, x_view);
758 m_exec_space.fence();
759 auto const apply_end = std::chrono::steady_clock::now();
761 m_apply_duration += std::chrono::duration<double>(apply_end - apply_start).count();
762 log_apply_progress(
"simple", m_apply_count, m_apply_duration);
766 gko::LinOp
const* alpha,
768 gko::LinOp
const* beta,
769 gko::LinOp* x)
const override
771 log_apply_start(
"advanced", m_advanced_apply_count + 1);
772 auto const apply_start = std::chrono::steady_clock::now();
773 auto const* alpha_dense =
dynamic_cast<dense_type const*
>(alpha);
774 auto const* b_dense =
dynamic_cast<dense_type const*
>(b);
775 auto const* beta_dense =
dynamic_cast<dense_type const*
>(beta);
776 auto* x_dense =
dynamic_cast<dense_type*
>(x);
777 if (alpha_dense ==
nullptr || b_dense ==
nullptr || beta_dense ==
nullptr
778 || x_dense ==
nullptr) {
779 throw std::invalid_argument(
780 "MatrixFreeLinOp expects dense alpha, beta, input, and output");
783 auto alpha_view = gko::ext::kokkos::map_data<memory_space>(*alpha_dense);
784 auto b_view = gko::ext::kokkos::map_data<memory_space>(*b_dense);
785 auto beta_view = gko::ext::kokkos::map_data<memory_space>(*beta_dense);
786 auto x_view = gko::ext::kokkos::map_data<memory_space>(*x_dense);
787 if (x_view.extent(0) != m_advanced_applied.extent(0)
788 || x_view.extent(1) != m_advanced_applied.extent(1)) {
789 throw std::invalid_argument(
"MatrixFreeLinOp received an unsupported dense shape");
791 auto const applied = m_advanced_applied;
792 if constexpr (workspace_traits::enabled) {
793 if constexpr (
requires(
794 OperatorModel
const& model,
796 decltype(b_view) input,
797 decltype(applied) output,
798 workspace_type& workspace) {
799 model.apply(ex, input, output, workspace);
801 if (uses_precomputed_matrix_free_stencils(*m_operator_model)) {
802 m_operator_model->apply(m_exec_space, b_view, applied);
804 if (m_workspace ==
nullptr) {
805 m_workspace = std::make_shared<workspace_type>(
806 m_operator_model->create_matrix_free_workspace(m_exec_space));
808 m_operator_model->apply(m_exec_space, b_view, applied, *m_workspace);
811 m_operator_model->apply(m_exec_space, b_view, applied);
814 m_operator_model->apply(m_exec_space, b_view, applied);
816 Kokkos::parallel_for(
817 "similie_matrix_free_linop_advanced_apply",
818 Kokkos::MDRangePolicy<
821 2>>(m_exec_space, {0, 0}, {x_view.extent(0), x_view.extent(1)}),
822 KOKKOS_LAMBDA(std::size_t row, std::size_t column) {
823 x_view(row, column) = alpha_view(0, 0) * applied(row, column)
824 + beta_view(0, 0) * x_view(row, column);
826 m_exec_space.fence();
827 auto const apply_end = std::chrono::steady_clock::now();
828 ++m_advanced_apply_count;
829 m_advanced_apply_duration += std::chrono::duration<double>(apply_end - apply_start).count();
830 log_apply_progress(
"advanced", m_advanced_apply_count, m_advanced_apply_duration);
833 void log_apply_start(
char const* apply_kind, std::size_t apply_count)
const
835 if (!env_flag_enabled(
"SIMILIE_MATRIX_FREE_APPLY_PROGRESS")) {
838 std::size_t
const stride =
static_cast<std::size_t
>(
839 std::max(1, env_int_or(
"SIMILIE_MATRIX_FREE_APPLY_PROGRESS_STRIDE", 10)));
840 if (apply_count == 1 || apply_count % stride == 0) {
841 std::cout <<
"SimiLie matrix-free apply start: kind=" << apply_kind
842 <<
" count=" << apply_count << std::endl;
846 void log_apply_progress(
char const* apply_kind, std::size_t apply_count,
double duration)
const
848 if (!env_flag_enabled(
"SIMILIE_MATRIX_FREE_APPLY_PROGRESS")) {
851 std::size_t
const stride =
static_cast<std::size_t
>(
852 std::max(1, env_int_or(
"SIMILIE_MATRIX_FREE_APPLY_PROGRESS_STRIDE", 10)));
853 if (apply_count == 1 || apply_count % stride == 0) {
854 std::cout <<
"SimiLie matrix-free apply progress: kind=" << apply_kind
855 <<
" count=" << apply_count <<
" cumulative_duration=" << duration
860 void log_timing()
const
862 if (!env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
865 std::cout <<
"SimiLie matrix-free operator timing: apply_count=" << m_apply_count
866 <<
" apply_duration=" << m_apply_duration
867 <<
" advanced_apply_count=" << m_advanced_apply_count
868 <<
" advanced_apply_duration=" << m_advanced_apply_duration <<
'\n';
872template <
class ExecSpace,
class OperatorModel,
class StateView>
873class StateDependentMatrixFreeLinOp
875 ,
public gko::EnableCloneable<
876 StateDependentMatrixFreeLinOp<ExecSpace, OperatorModel, StateView>>
878 using value_type = double;
879 using dense_type = gko::matrix::Dense<value_type>;
880 using memory_space =
typename ExecSpace::memory_space;
881 using base_type = gko::LinOp;
883 ExecSpace m_exec_space;
884 std::shared_ptr<OperatorModel const> m_operator_model;
886 mutable Kokkos::View<double**, Kokkos::LayoutRight, memory_space> m_advanced_applied;
889 static constexpr bool IS_SYMMETRIC = [] {
890 if constexpr (
requires { OperatorModel::IS_SYMMETRIC; }) {
891 return OperatorModel::IS_SYMMETRIC;
896 explicit StateDependentMatrixFreeLinOp(std::shared_ptr<gko::Executor const> exec)
897 : base_type(std::move(exec))
904 StateDependentMatrixFreeLinOp(
905 std::shared_ptr<gko::Executor const> exec,
906 ExecSpace
const& exec_space,
907 std::shared_ptr<OperatorModel const> operator_model,
908 StateView
const& state)
909 : base_type(exec, gko::dim<2>(operator_model->size(), operator_model->size()))
910 , m_exec_space(exec_space)
911 , m_operator_model(std::move(operator_model))
913 , m_advanced_applied(
914 "similie_state_dependent_linop_advanced_workspace",
915 m_operator_model->size(),
921 void apply_impl(gko::LinOp
const* b, gko::LinOp* x)
const override
923 auto const* b_dense =
dynamic_cast<dense_type const*
>(b);
924 auto* x_dense =
dynamic_cast<dense_type*
>(x);
925 if (b_dense ==
nullptr || x_dense ==
nullptr) {
926 throw std::invalid_argument(
927 "StateDependentMatrixFreeLinOp expects dense inputs and outputs");
929 auto b_view = gko::ext::kokkos::map_data<memory_space>(*b_dense);
930 auto x_view = gko::ext::kokkos::map_data<memory_space>(*x_dense);
931 apply_jacobian(m_exec_space, *m_operator_model, m_state, b_view, x_view);
935 gko::LinOp
const* alpha,
937 gko::LinOp
const* beta,
938 gko::LinOp* x)
const override
940 auto const* alpha_dense =
dynamic_cast<dense_type const*
>(alpha);
941 auto const* b_dense =
dynamic_cast<dense_type const*
>(b);
942 auto const* beta_dense =
dynamic_cast<dense_type const*
>(beta);
943 auto* x_dense =
dynamic_cast<dense_type*
>(x);
944 if (alpha_dense ==
nullptr || b_dense ==
nullptr || beta_dense ==
nullptr
945 || x_dense ==
nullptr) {
946 throw std::invalid_argument(
947 "StateDependentMatrixFreeLinOp expects dense alpha, beta, input, and output");
949 auto alpha_view = gko::ext::kokkos::map_data<memory_space>(*alpha_dense);
950 auto b_view = gko::ext::kokkos::map_data<memory_space>(*b_dense);
951 auto beta_view = gko::ext::kokkos::map_data<memory_space>(*beta_dense);
952 auto x_view = gko::ext::kokkos::map_data<memory_space>(*x_dense);
953 if (x_view.extent(0) != m_advanced_applied.extent(0)
954 || x_view.extent(1) != m_advanced_applied.extent(1)) {
955 throw std::invalid_argument(
956 "StateDependentMatrixFreeLinOp received an unsupported dense shape");
958 auto const applied = m_advanced_applied;
959 apply_jacobian(m_exec_space, *m_operator_model, m_state, b_view, applied);
960 Kokkos::parallel_for(
961 "similie_state_dependent_linop_advanced_apply",
962 Kokkos::MDRangePolicy<
965 2>>(m_exec_space, {0, 0}, {x_view.extent(0), x_view.extent(1)}),
966 KOKKOS_LAMBDA(std::size_t row, std::size_t column) {
967 x_view(row, column) = alpha_view(0, 0) * applied(row, column)
968 + beta_view(0, 0) * x_view(row, column);
970 m_exec_space.fence();
974inline auto build_jacobi_preconditioner_factory(
975 std::shared_ptr<gko::Executor const>
const& gko_exec,
976 StrongFormulationSolverSettings
const& settings)
978 return gko::preconditioner::Jacobi<double>::build()
979 .with_max_block_size(settings.jacobi_max_block_size)
983inline PreconditionerType selected_preconditioner(StrongFormulationSolverSettings
const& settings)
985 char const*
const value = std::getenv(
"SIMILIE_PRECONDITIONER");
986 if (value ==
nullptr || value[0] ==
'\0') {
987 if (env_flag_enabled(
"SIMILIE_DISABLE_JACOBI_PRECONDITIONER")) {
990 return settings.preconditioner;
995inline std::shared_ptr<gko::LinOp const> build_identity_preconditioner(
996 std::shared_ptr<gko::Executor const>
const& gko_exec,
999 return gko::matrix::Identity<double>::create(gko_exec, size);
1002inline std::shared_ptr<gko::LinOp const> build_preconditioner(
1003 std::shared_ptr<gko::Executor const>
const& gko_exec,
1004 std::shared_ptr<gko::LinOp const>
const& matrix,
1005 StrongFormulationSolverSettings
const& settings)
1008 if (!env_flag_enabled(
"SIMILIE_SOLVER_QUIET"))
1011 switch (preconditioner) {
1013 auto level = gko::multigrid::Pgm<double, gko::int32>::build().with_deterministic(
true).on(
1016 auto coarse = gko::solver::Ir<double>::build()
1017 .with_solver(build_jacobi_preconditioner_factory(gko_exec, settings))
1018 .with_relaxation_factor(0.8)
1019 .with_default_initial_guess(gko::solver::initial_guess_mode::zero)
1020 .with_criteria(gko::stop::Iteration::build().with_max_iters(20U))
1022 return gko::solver::Multigrid::build()
1023 .with_mg_level(std::move(level))
1024 .with_min_coarse_rows(32U)
1025 .with_max_levels(20U)
1026 .with_smoother_iters(2U)
1027 .with_smoother_relax(0.8)
1028 .with_post_uses_pre(
true)
1029 .with_coarsest_solver(std::move(coarse))
1030 .with_default_initial_guess(gko::solver::initial_guess_mode::zero)
1031 .with_criteria(gko::stop::Iteration::build().with_max_iters(1U))
1036 return build_identity_preconditioner(gko_exec, matrix->get_size()[0]);
1038 auto preconditioner_factory = build_jacobi_preconditioner_factory(gko_exec, settings);
1039 return std::shared_ptr<gko::LinOp const>(
1040 preconditioner_factory->generate(matrix).release());
1043 auto preconditioner_factory
1044 = gko::preconditioner::SpdIsai<double, gko::int32>::build().on(gko_exec);
1045 return std::shared_ptr<gko::LinOp const>(
1046 preconditioner_factory->generate(matrix).release());
1049 auto preconditioner_factory
1050 = gko::preconditioner::GeneralIsai<double, gko::int32>::build().on(gko_exec);
1051 return std::shared_ptr<gko::LinOp const>(
1052 preconditioner_factory->generate(matrix).release());
1055 auto preconditioner_factory = gko::preconditioner::GaussSeidel<double, gko::int32>::build()
1056 .with_symmetric(
true)
1058 return std::shared_ptr<gko::LinOp const>(
1059 preconditioner_factory->generate(matrix).release());
1062 auto preconditioner_factory = gko::preconditioner::Sor<double, gko::int32>::build()
1063 .with_symmetric(
true)
1064 .with_relaxation_factor(env_double_or(
1065 "SIMILIE_SOR_RELAXATION_FACTOR",
1066 settings.sor_relaxation_factor))
1068 return std::shared_ptr<gko::LinOp const>(
1069 preconditioner_factory->generate(matrix).release());
1072 auto jacobi_factory = build_jacobi_preconditioner_factory(gko_exec, settings);
1073 auto iterations_criterion = gko::stop::Iteration::build()
1074 .with_max_iters(settings.chebyshev_iterations)
1076 auto preconditioner_factory
1077 = gko::solver::Chebyshev<double>::build()
1078 .with_criteria(std::move(iterations_criterion))
1079 .with_preconditioner(std::move(jacobi_factory))
1080 .with_foci(settings.chebyshev_lower_bound, settings.chebyshev_upper_bound)
1081 .with_default_initial_guess(gko::solver::initial_guess_mode::zero)
1083 return std::shared_ptr<gko::LinOp const>(
1084 preconditioner_factory->generate(matrix).release());
1087 auto jacobi_factory = build_jacobi_preconditioner_factory(gko_exec, settings);
1088 auto iterations_criterion
1089 = gko::stop::Iteration::build().with_max_iters(settings.ir_iterations).on(gko_exec);
1090 auto preconditioner_factory
1091 = gko::solver::Ir<double>::build()
1092 .with_criteria(std::move(iterations_criterion))
1093 .with_solver(std::move(jacobi_factory))
1094 .with_relaxation_factor(settings.ir_relaxation_factor)
1095 .with_default_initial_guess(gko::solver::initial_guess_mode::zero)
1097 return std::shared_ptr<gko::LinOp const>(
1098 preconditioner_factory->generate(matrix).release());
1101 throw std::runtime_error(
"unsupported strong-formulation preconditioner");
1104template <
class ExecSpace,
class OperatorModel,
class RHSViewType,
class SolutionViewType>
1105StrongFormulationSolverDiagnostics solve_linearized_system(
1106 ExecSpace
const& exec_space,
1107 std::shared_ptr<gko::Executor const>
const& gko_exec,
1108 OperatorModel
const& operator_model,
1109 RHSViewType
const& rhs,
1110 SolutionViewType
const& solution,
1111 StrongFormulationSolverSettings
const& settings,
1112 std::shared_ptr<gko::LinOp const>
const& assembled_matrix,
1113 std::shared_ptr<gko::LinOp const>
const& preconditioner)
1115 StrongFormulationSolverDiagnostics diagnostics;
1116 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1117 std::cout <<
"SimiLie linear solve stage: computing initial residual norm" << std::endl;
1119 auto const initial_residual_start = std::chrono::steady_clock::now();
1120 if (!settings.use_initial_guess) {
1121 fill(exec_space, solution, 0.0);
1123 diagnostics.rhs_norm_l2 = residual_norm_l2(exec_space, rhs);
1124 diagnostics.initial_residual_l2 = diagnostics.rhs_norm_l2;
1125 if (settings.use_initial_guess && diagnostics.rhs_norm_l2 != 0.0) {
1126 using memory_space =
typename SolutionViewType::memory_space;
1127 Kokkos::View<double**, memory_space>
1128 residual(
"similie_initial_residual", rhs.extent(0), rhs.extent(1));
1129 if constexpr (std::is_base_of_v<gko::LinOp, OperatorModel>) {
1130 auto x = to_gko_dense(gko_exec, solution);
1131 auto applied = to_gko_dense(gko_exec, residual);
1132 assembled_matrix->apply(x.dense, applied.dense);
1133 gko_exec->synchronize();
1134 copy_back_from_gko_dense_bridge(residual, applied);
1136 operator_model.apply(exec_space, solution, residual);
1138 axpy_inplace(exec_space, residual, -1.0, rhs);
1139 diagnostics.initial_residual_l2 = residual_norm_l2(exec_space, residual);
1141 auto const initial_residual_end = std::chrono::steady_clock::now();
1142 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1143 std::cout <<
"SimiLie linear solve stage: initial residual norm computed duration="
1144 << std::chrono::duration<double>(initial_residual_end - initial_residual_start)
1148 diagnostics.final_residual_l2 = diagnostics.initial_residual_l2;
1149 diagnostics.final_relative_residual = diagnostics.initial_residual_l2 == 0.0 ? 0.0 : 1.0;
1150 if (diagnostics.rhs_norm_l2 == 0.0) {
1151 fill(exec_space, solution, 0.0);
1152 diagnostics.initial_residual_l2 = 0.0;
1155 if (diagnostics.initial_residual_l2 <= settings.relative_tolerance * diagnostics.rhs_norm_l2) {
1156 diagnostics.final_relative_residual
1157 = diagnostics.initial_residual_l2 / diagnostics.rhs_norm_l2;
1161 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1162 std::cout <<
"SimiLie linear solve stage: building solver factory" << std::endl;
1164 auto const solver_factory_start = std::chrono::steady_clock::now();
1165 auto residual_criterion = gko::stop::ResidualNorm<double>::build()
1166 .with_reduction_factor(settings.relative_tolerance)
1167 .with_baseline(gko::stop::mode::rhs_norm)
1169 auto iterations_criterion
1170 = gko::stop::Iteration::build().with_max_iters(settings.max_iterations).on(gko_exec);
1171 std::unique_ptr<gko::LinOpFactory> solver_factory;
1172 bool const nonsymmetric_operator = [] {
1173 if constexpr (
requires { OperatorModel::IS_SYMMETRIC; }) {
1174 return !OperatorModel::IS_SYMMETRIC;
1176 return std::is_base_of_v<gko::LinOp, OperatorModel>;
1178 bool const default_to_gmres = nonsymmetric_operator && std::getenv(
"SIMILIE_SOLVER") ==
nullptr;
1179 gko::size_type
const gmres_krylov_dim =
static_cast<gko::size_type
>(std::max(
1181 env_int_or(
"SIMILIE_GMRES_KRYLOV_DIM", nonsymmetric_operator ? 1500 : 100)));
1182 if (detail::env_value_equals(
"SIMILIE_SOLVER",
"minres")) {
1183 solver_factory = gko::solver::Minres<double>::build()
1184 .with_generated_preconditioner(preconditioner)
1186 std::move(residual_criterion),
1187 std::move(iterations_criterion))
1189 }
else if (detail::env_value_equals(
"SIMILIE_SOLVER",
"fcg")) {
1190 solver_factory = gko::solver::Fcg<double>::build()
1191 .with_generated_preconditioner(preconditioner)
1193 std::move(residual_criterion),
1194 std::move(iterations_criterion))
1196 }
else if (detail::env_value_equals(
"SIMILIE_SOLVER",
"gmres") || default_to_gmres) {
1197 solver_factory = gko::solver::Gmres<double>::build()
1198 .with_generated_preconditioner(preconditioner)
1200 std::move(residual_criterion),
1201 std::move(iterations_criterion))
1202 .with_krylov_dim(gmres_krylov_dim)
1204 }
else if (detail::env_value_equals(
"SIMILIE_SOLVER",
"bicgstab")) {
1205 solver_factory = gko::solver::Bicgstab<double>::build()
1206 .with_generated_preconditioner(preconditioner)
1208 std::move(residual_criterion),
1209 std::move(iterations_criterion))
1211 }
else if (detail::env_value_equals(
"SIMILIE_SOLVER",
"gcr")) {
1213 = gko::solver::Gcr<double>::build()
1214 .with_generated_preconditioner(preconditioner)
1216 std::move(residual_criterion),
1217 std::move(iterations_criterion))
1219 static_cast<gko::size_type
>(
1220 std::max(1, env_int_or(
"SIMILIE_GCR_KRYLOV_DIM", 100))))
1222 }
else if (detail::env_value_equals(
"SIMILIE_SOLVER",
"idr")) {
1224 = gko::solver::Idr<double>::build()
1225 .with_generated_preconditioner(preconditioner)
1227 std::move(residual_criterion),
1228 std::move(iterations_criterion))
1230 static_cast<gko::size_type
>(
1231 std::max(1, env_int_or(
"SIMILIE_IDR_SUBSPACE_DIM", 8))))
1232 .with_deterministic(
true)
1235 solver_factory = gko::solver::Cg<double>::build()
1236 .with_generated_preconditioner(preconditioner)
1238 std::move(residual_criterion),
1239 std::move(iterations_criterion))
1242 auto const solver_factory_end = std::chrono::steady_clock::now();
1243 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1244 std::cout <<
"SimiLie linear solve stage: solver factory built duration="
1245 << std::chrono::duration<double>(solver_factory_end - solver_factory_start)
1249 using MatrixFree = std::conditional_t<
1250 std::is_base_of_v<gko::LinOp, OperatorModel>,
1252 MatrixFreeLinOp<ExecSpace, OperatorModel>>;
1253 std::shared_ptr<MatrixFree const> matrix_free_system_matrix;
1254 std::shared_ptr<gko::LinOp const> system_matrix;
1255 if constexpr (std::is_base_of_v<gko::LinOp, OperatorModel>)
1256 system_matrix = assembled_matrix;
1257 else if (settings.use_matrix_free) {
1258 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1259 std::cout <<
"SimiLie linear solve stage: constructing matrix-free LinOp" << std::endl;
1261 auto const matrix_free_linop_start = std::chrono::steady_clock::now();
1262 auto operator_model_ptr
1263 = std::shared_ptr<OperatorModel const>(&operator_model, [](OperatorModel
const*) {
1265 matrix_free_system_matrix = std::make_shared<MatrixFreeLinOp<
1267 OperatorModel>>(gko_exec, exec_space, std::move(operator_model_ptr));
1268 auto const matrix_free_linop_end = std::chrono::steady_clock::now();
1269 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1270 std::cout <<
"SimiLie linear solve stage: matrix-free LinOp constructed duration="
1271 << std::chrono::duration<double>(
1272 matrix_free_linop_end - matrix_free_linop_start)
1276 system_matrix = matrix_free_system_matrix;
1278 system_matrix = assembled_matrix;
1280 if (settings.use_matrix_free && matrix_free_system_matrix !=
nullptr
1281 && assembled_matrix !=
nullptr && env_flag_enabled(
"SIMILIE_COMPARE_MATRIX_FREE_APPLY")) {
1282 using memory_space =
typename RHSViewType::memory_space;
1283 Kokkos::View<double**, memory_space> matrix_free_applied(
1284 "similie_compare_matrix_free_applied",
1287 Kokkos::View<double**, memory_space> assembled_applied(
1288 "similie_compare_assembled_applied",
1291 Kokkos::View<double**, memory_space>
1292 difference(
"similie_compare_apply_difference", rhs.extent(0), rhs.extent(1));
1293 auto probe_gko = to_gko_dense(gko_exec, rhs);
1294 auto matrix_free_applied_gko = to_gko_dense(gko_exec, matrix_free_applied);
1295 auto assembled_applied_gko = to_gko_dense(gko_exec, assembled_applied);
1296 matrix_free_system_matrix->apply(probe_gko.dense, matrix_free_applied_gko.dense);
1297 assembled_matrix->apply(probe_gko.dense, assembled_applied_gko.dense);
1298 gko_exec->synchronize();
1299 copy_back_from_gko_dense_bridge(matrix_free_applied, matrix_free_applied_gko);
1300 copy_back_from_gko_dense_bridge(assembled_applied, assembled_applied_gko);
1301 copy(exec_space, difference, matrix_free_applied);
1302 axpy_inplace(exec_space, difference, -1.0, assembled_applied);
1303 double const difference_norm = residual_norm_l2(exec_space, difference);
1304 double const assembled_norm = residual_norm_l2(exec_space, assembled_applied);
1305 std::cout <<
"SimiLie matrix-free apply comparison: difference_l2=" << difference_norm
1306 <<
" assembled_l2=" << assembled_norm <<
" relative_difference="
1307 << (assembled_norm == 0.0 ? 0.0 : difference_norm / assembled_norm) <<
'\n';
1308 if (rhs.extent(1) == 1 && rhs.extent(0) % 3 == 0) {
1309 auto const difference_host
1310 = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), difference);
1311 auto const assembled_host
1312 = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), assembled_applied);
1313 std::array<double, 3> component_difference_norms {};
1314 std::array<double, 3> component_assembled_norms {};
1315 for (std::size_t row = 0; row < rhs.extent(0); ++row) {
1316 std::size_t
const component = row % 3;
1317 component_difference_norms[component]
1318 += difference_host(row, 0) * difference_host(row, 0);
1319 component_assembled_norms[component]
1320 += assembled_host(row, 0) * assembled_host(row, 0);
1322 std::cout <<
"SimiLie matrix-free apply component comparison:";
1323 for (std::size_t component = 0; component < 3; ++component) {
1324 double const component_difference
1325 = std::sqrt(component_difference_norms[component]);
1326 double const component_assembled = std::sqrt(component_assembled_norms[component]);
1327 std::cout <<
" component" << component <<
"_difference_l2=" << component_difference
1328 <<
" component" << component <<
"_relative_difference="
1329 << (component_assembled == 0.0
1331 : component_difference / component_assembled);
1336 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1337 std::cout <<
"SimiLie linear solve stage: generating solver" << std::endl;
1339 auto const solver_generate_start = std::chrono::steady_clock::now();
1340 auto solver = solver_factory->generate(system_matrix);
1341 auto const solver_generate_end = std::chrono::steady_clock::now();
1342 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1343 std::cout <<
"SimiLie linear solve stage: solver generated duration="
1344 << std::chrono::duration<double>(solver_generate_end - solver_generate_start)
1348 auto convergence_logger = std::shared_ptr<gko::log::Convergence<double>>(
1349 gko::log::Convergence<double>::create().release());
1350 solver->add_logger(convergence_logger);
1351 std::shared_ptr<SolverProgressLogger> progress_logger;
1352 if (
unsigned int const progress_stride = solver_progress_stride(); progress_stride != 0U) {
1353 progress_logger = std::make_shared<SolverProgressLogger>(
1354 gko_exec->get_master(),
1355 diagnostics.initial_residual_l2,
1357 solver->add_logger(progress_logger);
1360 auto rhs_gko = to_gko_dense(gko_exec, rhs);
1361 auto solution_gko = to_gko_dense(gko_exec, solution);
1362 auto const optimization_start = std::chrono::steady_clock::now();
1363 if (env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1364 std::cout <<
"SimiLie linear solve stage: applying solver" << std::endl;
1366 solver->apply(rhs_gko.dense, solution_gko.dense);
1367 gko_exec->synchronize();
1368 copy_back_from_gko_dense_bridge(solution, solution_gko);
1369 auto const optimization_end = std::chrono::steady_clock::now();
1370 diagnostics.duration
1371 = std::chrono::duration<double>(optimization_end - optimization_start).count();
1372 if constexpr (!std::is_base_of_v<gko::LinOp, OperatorModel>)
1373 if (matrix_free_system_matrix !=
nullptr)
1374 matrix_free_system_matrix->log_timing();
1375 if (progress_logger !=
nullptr) {
1376 solver->remove_logger(progress_logger);
1378 solver->remove_logger(convergence_logger);
1379 diagnostics.iterations =
static_cast<unsigned int>(convergence_logger->get_num_iterations());
1380 diagnostics.converged = convergence_logger->has_converged();
1381 auto residual_norm_dense =
dynamic_cast<gko::matrix::Dense<double> const*
>(
1382 convergence_logger->get_residual_norm());
1383 bool const check_final_residual = env_value_equals(
"SIMILIE_SOLVER",
"minres")
1384 || env_value_equals(
"SIMILIE_SOLVER",
"bicgstab")
1385 || env_value_equals(
"SIMILIE_SOLVER",
"gcr")
1386 || env_value_equals(
"SIMILIE_SOLVER",
"idr");
1387 if (residual_norm_dense !=
nullptr && !check_final_residual) {
1388 auto host_dense = gko::matrix::Dense<
1389 double>::create(gko_exec->get_master(), residual_norm_dense->get_size());
1390 residual_norm_dense->convert_to(host_dense.get());
1391 diagnostics.final_residual_l2 = host_dense->at(0, 0);
1396 using memory_space =
typename SolutionViewType::memory_space;
1397 Kokkos::View<double**, memory_space>
1398 residual(
"similie_final_residual", rhs.extent(0), rhs.extent(1));
1399 auto residual_gko = to_gko_dense(gko_exec, residual);
1400 system_matrix->apply(solution_gko.dense, residual_gko.dense);
1401 gko_exec->synchronize();
1402 copy_back_from_gko_dense_bridge(residual, residual_gko);
1403 axpy_inplace(exec_space, residual, -1.0, rhs);
1404 diagnostics.final_residual_l2 = residual_norm_l2(exec_space, residual);
1406 diagnostics.final_relative_residual
1407 = diagnostics.rhs_norm_l2 == 0.0
1409 : diagnostics.final_residual_l2 / diagnostics.rhs_norm_l2;
1410 if (env_flag_enabled(
"SIMILIE_TRUE_RESIDUAL_DIAGNOSTICS")) {
1411 using memory_space =
typename SolutionViewType::memory_space;
1412 Kokkos::View<double**, memory_space>
1413 applied(
"similie_true_residual_applied", rhs.extent(0), rhs.extent(1));
1414 Kokkos::View<double**, memory_space>
1415 true_residual(
"similie_true_residual", rhs.extent(0), rhs.extent(1));
1416 auto solution_gko_for_residual = to_gko_dense(gko_exec, solution);
1417 auto applied_gko_for_residual = to_gko_dense(gko_exec, applied);
1418 system_matrix->apply(solution_gko_for_residual.dense, applied_gko_for_residual.dense);
1419 gko_exec->synchronize();
1420 copy_back_from_gko_dense_bridge(applied, applied_gko_for_residual);
1421 copy(exec_space, true_residual, rhs);
1422 axpy_inplace(exec_space, true_residual, -1.0, applied);
1423 double const true_residual_l2 = residual_norm_l2(exec_space, true_residual);
1424 double const true_relative_residual
1425 = diagnostics.rhs_norm_l2 == 0.0 ? 0.0 : true_residual_l2 / diagnostics.rhs_norm_l2;
1426 std::cout <<
"SimiLie true residual diagnostics: residual_l2=" << true_residual_l2
1427 <<
" relative_residual=" << true_relative_residual <<
'\n';
1439template <
class ExecSpace,
class RHSView,
class SolutionView>
1441 ExecSpace
const& exec,
1442 std::shared_ptr<gko::matrix::Csr<double, gko::int32>>
const& matrix,
1443 std::vector<RHSView>
const& rhs,
1444 std::vector<SolutionView>
const& solutions,
1447 if (!matrix || rhs.empty() || rhs.size() != solutions.size())
1448 throw std::invalid_argument(
"invalid assembled linear system batch");
1449 auto const executor = matrix->get_executor();
1450 if (detail::env_flag_enabled(
"SIMILIE_MATRIX_DIAGNOSTICS")) {
1451 gko::matrix_data<double, gko::int32> data;
1452 matrix->write(data);
1453 detail::log_matrix_diagnostics(data);
1455 auto const preconditioner = detail::build_preconditioner(executor, matrix, settings);
1456 settings.use_matrix_free =
false;
1457 if (!detail::env_flag_enabled(
"SIMILIE_SOLVER_QUIET"))
1458 std::cout <<
"SimiLie linear system mode: assembled tensor stencils" <<
'\n';
1459 std::vector<StrongFormulationSolverDiagnostics> diagnostics;
1460 for (std::size_t i = 0; i < rhs.size(); ++i) {
1461 if (rhs[i].extent(0) != matrix->get_size()[0] || rhs[i].extent(1) != 1
1462 || solutions[i].extent(0) != rhs[i].extent(0) || solutions[i].extent(1) != 1)
1463 throw std::invalid_argument(
"assembled linear system shape mismatch");
1464 diagnostics.push_back(
1465 detail::solve_linearized_system(
1474 if (!std::isfinite(diagnostics.back().final_relative_residual)
1475 || diagnostics.back().final_relative_residual > settings.relative_tolerance * 10.0)
1476 throw std::runtime_error(
"assembled linear solve did not converge");
1481template <
class ExecSpace,
class OperatorModel,
class RHSViewType,
class SolutionViewType>
1483 ExecSpace
const& exec_space,
1484 OperatorModel
const& operator_model,
1485 RHSViewType
const& rhs,
1486 SolutionViewType
const& solution,
1489 StrongFormulationSolverDiagnostics diagnostics;
1491 if (!settings.use_initial_guess || !OperatorModel::IS_LINEAR) {
1492 detail::fill(exec_space, solution, 0.0);
1494 auto const gko_exec = settings.generated_preconditioner
1495 ? settings.generated_preconditioner->get_executor()
1496 : gko::ext::kokkos::create_executor(exec_space);
1497 if constexpr (OperatorModel::IS_LINEAR) {
1498 if (!detail::env_flag_enabled(
"SIMILIE_SOLVER_QUIET"))
1499 std::cout <<
"SimiLie linear system mode: "
1500 << (settings.use_matrix_free ?
"matrix-free" :
"assembled") <<
'\n';
1501 diagnostics.initial_residual_l2 = detail::residual_norm_l2(exec_space, rhs);
1502 diagnostics.final_residual_l2 = diagnostics.initial_residual_l2;
1503 diagnostics.final_relative_residual = diagnostics.initial_residual_l2 == 0.0 ? 0.0 : 1.0;
1504 if (diagnostics.initial_residual_l2 == 0.0) {
1505 detail::fill(exec_space, solution, 0.0);
1508 auto const matrix_build_start = std::chrono::steady_clock::now();
1509 std::shared_ptr<gko::matrix::Csr<double, gko::int32>> matrix;
1510 auto const matrix_build_end = std::chrono::steady_clock::now();
1511 std::shared_ptr<gko::LinOp const> preconditioner;
1512 PreconditionerType const preconditioner_type = detail::selected_preconditioner(settings);
1513 if (settings.generated_preconditioner) {
1514 preconditioner = settings.generated_preconditioner;
1515 if (!settings.use_matrix_free)
1516 matrix = detail::build_matrix(gko_exec, operator_model);
1519 if (!detail::env_flag_enabled(
"SIMILIE_SOLVER_QUIET"))
1520 std::cout <<
"SimiLie Ginkgo preconditioner: "
1522 preconditioner = detail::build_identity_preconditioner(gko_exec, operator_model.size());
1524 matrix = detail::build_matrix(gko_exec, operator_model);
1525 auto const actual_matrix_build_end = std::chrono::steady_clock::now();
1526 preconditioner = detail::build_preconditioner(
1528 std::static_pointer_cast<gko::LinOp const>(matrix),
1530 if (detail::env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1531 std::cout <<
"SimiLie linear setup timing: matrix_build_duration="
1532 << std::chrono::duration<double>(
1533 actual_matrix_build_end - matrix_build_start)
1535 <<
" preconditioner_build_duration="
1536 << std::chrono::duration<double>(
1537 std::chrono::steady_clock::now() - actual_matrix_build_end)
1541 return detail::solve_linearized_system(
1548 std::static_pointer_cast<gko::LinOp const>(matrix),
1551 auto const preconditioner_build_end = std::chrono::steady_clock::now();
1552 if (detail::env_flag_enabled(
"SIMILIE_SOLVER_TIMING")) {
1553 std::cout <<
"SimiLie linear setup timing: matrix_build_duration="
1554 << std::chrono::duration<double>(matrix_build_end - matrix_build_start)
1556 <<
" preconditioner_build_duration="
1557 << std::chrono::duration<double>(preconditioner_build_end - matrix_build_end)
1561 return detail::solve_linearized_system(
1568 std::static_pointer_cast<gko::LinOp const>(matrix),
1571 using memory_space =
typename SolutionViewType::memory_space;
1572 Kokkos::View<double**, memory_space>
1573 residual(
"similie_nonlinear_residual", rhs.extent(0), rhs.extent(1));
1574 Kokkos::View<double**, memory_space>
1575 operator_value(
"similie_nonlinear_operator_value", rhs.extent(0), rhs.extent(1));
1576 Kokkos::View<double**, memory_space>
1577 correction_rhs(
"similie_nonlinear_correction_rhs", rhs.extent(0), rhs.extent(1));
1578 Kokkos::View<double**, memory_space>
1579 delta(
"similie_nonlinear_delta", rhs.extent(0), rhs.extent(1));
1580 Kokkos::View<double**, memory_space> candidate_solution(
1581 "similie_nonlinear_candidate_solution",
1584 Kokkos::View<double**, memory_space> candidate_residual(
1585 "similie_nonlinear_candidate_residual",
1588 Kokkos::View<double**, memory_space> candidate_operator_value(
1589 "similie_nonlinear_candidate_operator_value",
1592 auto const optimization_start = std::chrono::steady_clock::now();
1593 diagnostics.converged =
false;
1595 operator_model.apply(exec_space, solution, operator_value);
1596 detail::copy(exec_space, residual, rhs);
1597 detail::axpy_inplace(exec_space, residual, -1.0, operator_value);
1598 diagnostics.initial_residual_l2 = detail::residual_norm_l2(exec_space, residual);
1599 diagnostics.final_residual_l2 = diagnostics.initial_residual_l2;
1600 diagnostics.final_relative_residual = diagnostics.initial_residual_l2 == 0.0 ? 0.0 : 1.0;
1601 if (diagnostics.initial_residual_l2 == 0.0) {
1602 diagnostics.converged =
true;
1606 constexpr unsigned int NONLINEAR_MAX_ITERS = 30U;
1607 for (
unsigned int iteration = 0; iteration < NONLINEAR_MAX_ITERS; ++iteration) {
1608 diagnostics.final_residual_l2 = detail::residual_norm_l2(exec_space, residual);
1609 diagnostics.final_relative_residual
1610 = diagnostics.initial_residual_l2 == 0.0
1612 : diagnostics.final_residual_l2 / diagnostics.initial_residual_l2;
1613 detail::log_nonlinear_progress(iteration, diagnostics);
1614 if (!std::isfinite(diagnostics.final_residual_l2)) {
1617 if (diagnostics.final_relative_residual <= settings.relative_tolerance) {
1618 diagnostics.converged =
true;
1622 auto matrix = detail::build_matrix(gko_exec, operator_model, solution);
1623 auto preconditioner = detail::build_preconditioner(
1625 std::static_pointer_cast<gko::LinOp const>(matrix),
1628 detail::copy(exec_space, correction_rhs, residual);
1629 StrongFormulationSolverSettings correction_settings = settings;
1630 correction_settings.use_initial_guess =
false;
1631 if (settings.use_matrix_free) {
1632 auto operator_model_ptr = std::shared_ptr<
1633 OperatorModel
const>(&operator_model, [](OperatorModel
const*) {});
1634 auto system_matrix = std::make_shared<detail::StateDependentMatrixFreeLinOp<
1637 SolutionViewType>>(gko_exec, exec_space, operator_model_ptr, solution);
1638 auto const inner = detail::solve_linearized_system(
1644 correction_settings,
1647 diagnostics.iterations += inner.iterations;
1649 auto inner = detail::solve_linearized_system(
1655 correction_settings,
1656 std::static_pointer_cast<gko::LinOp const>(matrix),
1658 diagnostics.iterations += inner.iterations;
1660 double const current_residual_l2 = detail::residual_norm_l2(exec_space, residual);
1661 if (!std::isfinite(detail::residual_norm_l2(exec_space, delta))) {
1665 double best_alpha = 0.0;
1666 double best_residual_l2 = std::numeric_limits<double>::infinity();
1667 unsigned int const max_line_search_steps =
static_cast<unsigned int>(
1668 std::max(1, detail::env_int_or(
"SIMILIE_NONLINEAR_LINE_SEARCH_STEPS", 8)));
1669 double const line_search_reduction = std::
1670 clamp(detail::env_double_or(
"SIMILIE_NONLINEAR_LINE_SEARCH_REDUCTION", 0.5),
1673 for (
unsigned int line_search_step = 0; line_search_step < max_line_search_steps;
1674 ++line_search_step) {
1675 detail::copy(exec_space, candidate_solution, solution);
1676 detail::axpy_inplace(exec_space, candidate_solution, alpha, delta);
1677 operator_model.apply(exec_space, candidate_solution, candidate_operator_value);
1678 detail::copy(exec_space, candidate_residual, rhs);
1679 detail::axpy_inplace(
1683 candidate_operator_value);
1684 double const candidate_residual_l2
1685 = detail::residual_norm_l2(exec_space, candidate_residual);
1686 if (candidate_residual_l2 < best_residual_l2) {
1688 best_residual_l2 = candidate_residual_l2;
1690 if (candidate_residual_l2 < current_residual_l2) {
1693 alpha *= line_search_reduction;
1697 if (!std::isfinite(best_residual_l2) || best_residual_l2 >= current_residual_l2) {
1700 if (best_alpha != alpha) {
1702 detail::copy(exec_space, candidate_solution, solution);
1703 detail::axpy_inplace(exec_space, candidate_solution, alpha, delta);
1704 operator_model.apply(exec_space, candidate_solution, candidate_operator_value);
1705 detail::copy(exec_space, candidate_residual, rhs);
1706 detail::axpy_inplace(
1710 candidate_operator_value);
1711 best_residual_l2 = detail::residual_norm_l2(exec_space, candidate_residual);
1713 if (detail::solver_progress_enabled()) {
1714 std::cout <<
"SimiLie nonlinear line search: iteration=" << iteration
1715 <<
" alpha=" << alpha <<
" residual_l2=" << best_residual_l2
1716 <<
" previous_residual_l2=" << current_residual_l2 << std::endl;
1718 detail::copy(exec_space, solution, candidate_solution);
1719 detail::copy(exec_space, operator_value, candidate_operator_value);
1720 detail::copy(exec_space, residual, candidate_residual);
1722 auto const optimization_end = std::chrono::steady_clock::now();
1723 diagnostics.duration
1724 = std::chrono::duration<double>(optimization_end - optimization_start).count();
1725 diagnostics.final_residual_l2 = detail::residual_norm_l2(exec_space, residual);
1726 diagnostics.final_relative_residual
1727 = diagnostics.initial_residual_l2 == 0.0
1729 : diagnostics.final_residual_l2 / diagnostics.initial_residual_l2;