15#include <similie/exterior/coboundary/coboundary.hpp>
16#include <similie/tensor/tensor_indices/diagonal_tensor.hpp>
17#include <similie/tensor/tensor_indices/scaling_tensor.hpp>
19#include "stencil_indices.hpp"
25template <
class Index,
class Source,
class Target, std::
size_t Radius>
30 Target::mem_size() * Source::mem_size() * detail::StencilShape<Index, Radius>::size>
32 std::array<double, Target::mem_size()>
offset {};
38template <
class Index,
class Source,
class Target, std::
size_t Radius>
52 std::size_t FirstRadius,
53 std::size_t SecondRadius>
57 = detail::StencilShape<Index, FirstRadius>::size * First::mem_size();
59 = detail::StencilShape<Index, SecondRadius>::size * Second::mem_size();
67template <
class Memory,
class... Indices>
68KOKKOS_FUNCTION
auto coefficient_stencil(ddc::DiscreteDomain<Indices...>
const& domain)
70 std::array<double, (Indices::mem_size() * ... * std::size_t(1))> values {};
71 tensor::Tensor<double, ddc::DiscreteDomain<Indices...>, Kokkos::layout_right, Memory> view(
74 ddc::DiscreteDomain<Indices...>,
76 Memory>(values.data(), domain));
81template <
class Source,
class Target>
82struct LocalOperatorStencil
84 std::array<double, Source::mem_size() * Target::mem_size()> weights {};
92 bool NormalizeSource =
true,
94 std::size_t SourceRank,
95 std::size_t TargetRank>
96KOKKOS_FUNCTION
double local_stencil_coefficient(
97 Stencil
const& stencil,
98 std::array<std::size_t, SourceRank> source,
99 std::array<std::size_t, TargetRank> target)
102 for (std::size_t i = 0; i < source.size(); ++i)
103 for (std::size_t j = i + 1; j < source.size(); ++j) {
104 if (source[i] == source[j])
106 if (source[i] > source[j]) {
107 Kokkos::kokkos_swap(source[i], source[j]);
111 for (std::size_t i = 0; i < target.size(); ++i)
112 for (std::size_t j = i + 1; j < target.size(); ++j) {
113 if (target[i] == target[j])
115 if (target[i] > target[j]) {
116 Kokkos::kokkos_swap(target[i], target[j]);
121 if constexpr (NormalizeSource)
122 for (std::size_t i = 2; i <= source.size(); ++i)
126 [stencil_component_index<Target>(target) * Source::mem_size()
127 + stencil_component_index<Source>(source)]
132template <std::
size_t Rank,
class Domain,
class Memory>
136 typename Domain::discrete_element_type {}));
137 Kokkos::View<Basis, Memory> m_basis;
140 using simplex_type =
typename Basis::simplex_type;
141 using memory_space = Memory;
142 StencilChain() : m_basis(Kokkos::view_alloc(Kokkos::WithoutInitializing,
"stencil_topology"))
146 Kokkos::deep_copy(m_basis, basis);
148 KOKKOS_FORCEINLINE_FUNCTION std::size_t size()
const
150 return m_basis().size();
152 KOKKOS_FORCEINLINE_FUNCTION simplex_type
const& operator[](std::size_t i)
const
156 KOKKOS_FORCEINLINE_FUNCTION
auto begin()
const
158 return m_basis().begin();
160 KOKKOS_FORCEINLINE_FUNCTION
auto end()
const
162 return m_basis().end();
166template <std::
size_t Rank,
class Domain,
class Memory>
169 StencilChain<Rank, Domain, Memory> chain;
171 (Rank < Domain::rank()),
172 StencilChains<Rank + 1, Domain, Memory>,
175 template <std::
size_t Selected>
176 KOKKOS_FORCEINLINE_FUNCTION
auto get()
const
178 if constexpr (Selected == Rank)
181 return tail.template get<Selected>();
185template <
class Domain,
class Element>
186KOKKOS_FORCEINLINE_FUNCTION std::size_t stencil_node(Domain
const& domain, Element element)
188 std::size_t node = 0;
189 for (std::size_t axis = 0; axis < Domain::rank(); ++axis)
190 node = node * ddc::detail::array(domain.extents())[axis] + ddc::detail::array(element)[axis]
191 - ddc::detail::array(domain.front())[axis];
195template <
class Domain>
196KOKKOS_FORCEINLINE_FUNCTION
typename Domain::discrete_element_type stencil_element(
197 Domain
const& domain,
200 typename Domain::discrete_element_type element = domain.front();
201 for (std::size_t axis = Domain::rank(); axis-- > 0;) {
202 ddc::detail::array(element)[axis] += node % ddc::detail::array(domain.extents())[axis];
203 node /= ddc::detail::array(domain.extents())[axis];
208template <
class Index,
class... Dimensions>
209constexpr std::array<bool,
sizeof...(Dimensions)> stencil_axes(ddc::DiscreteDomain<Dimensions...>)
211 return {ddc::type_seq_contains_v<
212 ddc::TypeSeq<typename Dimensions::continuous_dimension_type>,
213 typename Index::type_seq_dimensions>...};
218template <
class Index, std::
size_t Radius,
class Domain,
class Element>
219KOKKOS_FORCEINLINE_FUNCTION Element
220stencil_source(Domain
const& domain, Element element, std::size_t point)
222 auto const offset = StencilShape<Index, Radius>::offset(point);
223 auto const active = stencil_axes<Index>(Domain {});
224 std::size_t physical_axis = 0;
225 for (std::size_t axis = 0; axis < Domain::rank(); ++axis) {
228 std::ptrdiff_t
const front = ddc::detail::array(domain.front())[axis];
229 std::ptrdiff_t
const count = ddc::detail::array(domain.extents())[axis];
230 std::ptrdiff_t position = std::ptrdiff_t(ddc::detail::array(element)[axis]) - front
231 + offset[physical_axis++];
232 if (position < 0 || position >= count)
233 position = (position % count + count) % count;
234 ddc::detail::array(element)[axis] = front + position;
247KOKKOS_FORCEINLINE_FUNCTION Stencil<Index, Source, Target, Radius> operator_stencil(
248 Domain
const& domain,
250 Evaluate
const& evaluate)
252 Stencil<Index, Source, Target, Radius> stencil;
253 auto baseline = evaluate([&](
auto sample,
auto) {
255 for (std::size_t point = 0; point < StencilShape<Index, Radius>::size; ++point)
256 found |= stencil_source<Index, Radius>(domain, element, point) == sample;
257 stencil.complete &= found;
260 for (std::size_t target = 0; target < Target::mem_size(); ++target)
261 stencil.offset[target] = baseline.mem(ddc::DiscreteElement<Target>(target));
262 for (std::size_t source = 0; source < Source::mem_size(); ++source) {
263 auto response = evaluate([&](
auto,
auto component) {
264 return component.template uid<Source>() == source ? 1.0 : 0.0;
266 for (std::size_t target = 0; target < Target::mem_size(); ++target)
267 stencil.constant_response[target * Source::mem_size() + source]
268 = response.mem(ddc::DiscreteElement<Target>(target)) - stencil.offset[target];
270 for (std::size_t point = 0; point < StencilShape<Index, Radius>::size; ++point) {
271 Element
const sample = stencil_source<Index, Radius>(domain, element, point);
272 bool duplicate =
false;
273 for (std::size_t earlier = 0; earlier < point; ++earlier)
274 duplicate |= stencil_source<Index, Radius>(domain, element, earlier) == sample;
277 for (std::size_t source = 0; source < Source::mem_size(); ++source) {
278 auto response = evaluate([&](
auto sampled_element,
auto component) {
279 return sampled_element == sample && component.template uid<Source>() == source
283 for (std::size_t target = 0; target < Target::mem_size(); ++target)
285 [(target * StencilShape<Index, Radius>::size + point) * Source::mem_size()
287 = response.mem(ddc::DiscreteElement<Target>(target))
288 - stencil.offset[target];
295template <
class Index,
class Domain,
class Memory,
class Evaluate>
302 using memory_space = Memory;
303 using non_indices_domain_t = Domain;
304 using indices_domain_t = ddc::DiscreteDomain<Index>;
305 KOKKOS_FORCEINLINE_FUNCTION StencilField(Domain
const& domain, Evaluate
const& evaluate)
307 , m_evaluate(evaluate)
310 KOKKOS_FORCEINLINE_FUNCTION Domain non_indices_domain()
const
314 KOKKOS_FORCEINLINE_FUNCTION
static tensor::TensorAccessor<Index> accessor()
318 template <
class Element,
class Component>
319 KOKKOS_FORCEINLINE_FUNCTION
double mem(Element element, Component component)
const
321 return m_evaluate(element, component);
323 template <
class Element>
324 KOKKOS_FORCEINLINE_FUNCTION
auto operator[](Element element)
const
326 return tensor::detail::pointwise_tensor<indices_domain_t, Memory>(
327 [&](
auto component) {
return mem(element, component); });
331template <
class Index,
class Source,
class Target, std::
size_t Radius,
class Domain,
class Memory>
336 double * [Target::mem_size() * StencilShape<Index, Radius>::size * Source::mem_size()],
340 Kokkos::View<
double * [Target::mem_size()], Kokkos::LayoutLeft, Memory> offset;
341 Kokkos::View<
double * [Target::mem_size()][Source::mem_size()], Kokkos::LayoutLeft, Memory>
346template <std::
size_t Component>
351template <
class Form,
bool Row,
class Sequence = std::make_index_sequence<Form::mem_size()>>
352struct LocalComponentIndex;
354template <
class Form,
bool Row, std::size_t... Components>
355struct LocalComponentIndex<
Form, Row, std::index_sequence<Components...>>
356 : tensor::TensorNaturalIndex<LocalComponent<Components>...>
360template <
class Index>
361concept LocalDiagonalCoefficientIndex = misc::Specialization<Index, tensor::TensorDiagonalIndex>
362 || misc::Specialization<Index, tensor::TensorScalingIndex>;
364template <
class Source,
class Target,
template <
class...>
class Coefficients>
365using LocalCoefficientIndex
366 = Coefficients<LocalComponentIndex<Target, true>, LocalComponentIndex<Source, false>>;
373 template <
class...>
class Coefficients = tensor::TensorFullIndex>
374struct LocalStencilStorage
378 double * [LocalCoefficientIndex<Source, Target, Coefficients>::mem_size()],
388 template <
class...>
class Coefficients = tensor::TensorFullIndex,
392LocalStencilStorage<Source, Target, Domain, Memory, Coefficients> cache_local_stencils(
395 Evaluate
const& evaluate)
397 using Index = LocalCoefficientIndex<Source, Target, Coefficients>;
399 misc::Specialization<Index, tensor::TensorFullIndex>
400 || misc::Specialization<Index, tensor::TensorDiagonalIndex>
401 || misc::Specialization<Index, tensor::TensorScalingIndex>);
402 if constexpr (LocalDiagonalCoefficientIndex<Index>)
403 static_assert(Source::mem_size() == Target::mem_size());
404 LocalStencilStorage<Source, Target, Domain, Memory, Coefficients> storage;
405 storage.domain = domain;
406 storage.weights =
decltype(storage.weights)(
407 Kokkos::view_alloc(Kokkos::WithoutInitializing,
"staged_local_stencils"),
409 auto const weights = storage.weights;
410 Kokkos::parallel_for(
411 "similie_cache_local_stencils",
412 Kokkos::RangePolicy<Exec>(exec, 0, domain.size()),
413 KOKKOS_LAMBDA(std::size_t node) {
414 auto const stencil = evaluate(stencil_element(domain, node));
416 if constexpr (LocalDiagonalCoefficientIndex<Index>) {
418 for (
double coefficient : stencil.weights)
419 scale = Kokkos::max(scale, Kokkos::abs(coefficient));
420 for (std::size_t row = 0; row < Target::mem_size(); ++row)
421 for (std::size_t col = 0; col < Source::mem_size(); ++col)
423 || Kokkos::abs(stencil.weights[row * Source::mem_size() + col])
424 <= 128.0 * std::numeric_limits<double>::epsilon()
426 &&
"Diagonal staged operator requires diagonal coefficients");
427 if constexpr (misc::Specialization<Index, tensor::TensorScalingIndex>) {
429 for (std::size_t component = 1; component < Source::mem_size(); ++component)
432 [component * Source::mem_size() + component]
433 - stencil.weights[0])
434 <= Kokkos::sqrt(std::numeric_limits<double>::epsilon()) * scale)
435 &&
"Scaling staged operator requires equal diagonal "
440 for (std::size_t entry = 0; entry < Index::mem_size(); ++entry) {
441 auto const ids = Index::mem_id_to_canonical_natural_ids(entry);
442 weights(node, entry) = stencil.weights[ids[0] * Source::mem_size() + ids[1]];
452template <
class Operator,
class Geometry,
class Input,
class Element,
class Rule1,
class Rule2>
453KOKKOS_FORCEINLINE_FUNCTION
auto stencil_value(
454 Geometry
const& geometry,
460 if constexpr (std::is_same_v<Rule2, NoStencilRule>)
461 return Operator::value(geometry, input, element, rule1);
463 return Operator::value(geometry, input, element, rule1, rule2);
476 class Rule2 = NoStencilRule>
482 typename Tensor::non_indices_domain_t,
483 typename Tensor::memory_space>
487 Geometry
const& geometry,
489 Rule2
const& rule2 = {})
491 using Domain =
typename Tensor::non_indices_domain_t;
492 using Memory =
typename Tensor::memory_space;
493 StencilStorage<Index, Source, Target, Radius, Domain, Memory> storage;
494 storage.domain = input.non_indices_domain();
495 storage.weights =
decltype(storage.weights)(
496 Kokkos::view_alloc(Kokkos::WithoutInitializing,
"staged_stencils"),
497 storage.domain.size());
498 storage.offset =
decltype(storage.offset)(
499 Kokkos::view_alloc(Kokkos::WithoutInitializing,
"staged_offsets"),
500 storage.domain.size());
501 storage.constant_response =
decltype(storage.constant_response)(
502 Kokkos::view_alloc(Kokkos::WithoutInitializing,
"staged_constant_response"),
503 storage.domain.size());
504 Kokkos::View<int, Memory> incomplete(
"staged_stencil_incomplete");
505 auto const weights = storage.weights;
506 auto const offset = storage.offset;
507 auto const constant_response = storage.constant_response;
508 Domain
const domain = storage.domain;
509 Kokkos::parallel_for(
510 "similie_cache_operator_stencils",
511 Kokkos::RangePolicy<Exec>(exec, 0, domain.size()),
512 KOKKOS_LAMBDA(std::size_t node) {
513 auto const stencil = stencil_value<
514 Operator>(geometry, input, stencil_element(domain, node), rule1, rule2);
515 if (!stencil.complete)
516 Kokkos::atomic_max(&incomplete(), 1);
517 for (std::size_t target = 0; target < Target::mem_size(); ++target) {
518 offset(node, target) = stencil.offset[target];
519 for (std::size_t source = 0; source < Source::mem_size(); ++source)
520 constant_response(node, target, source)
521 = stencil.constant_response[target * Source::mem_size() + source];
523 for (std::size_t entry = 0; entry < stencil.weights.size(); ++entry)
524 weights(node, entry) = stencil.weights[entry];
527 Kokkos::deep_copy(failed, incomplete);
530 throw std::invalid_argument(
531 "Boundary closure exceeds the operator's fixed stencil support");
545Output apply_stencils(
547 Output
const& output,
549 StencilStorage<Index, Source, Target, Radius, Domain, Memory>
const& storage)
551 if (input.non_indices_domain() != storage.domain)
552 throw std::invalid_argument(
"Staged operator input domain changed");
553 auto const weights = storage.weights;
554 auto const offset = storage.offset;
555 auto const constant_response = storage.constant_response;
556 Domain
const domain = storage.domain;
557 Domain
const output_domain = output.non_indices_domain();
558 if (!output_domain.empty()
561 throw std::invalid_argument(
"Staged operator output domain exceeds its cached domain");
562 Kokkos::parallel_for(
563 "similie_apply_staged_stencil",
564 Kokkos::RangePolicy<Exec>(exec, 0, output_domain.size() * Target::mem_size()),
565 KOKKOS_LAMBDA(std::size_t row) {
566 auto const element = stencil_element(output_domain, row / Target::mem_size());
567 std::size_t
const node = stencil_node(domain, element),
568 target = row % Target::mem_size();
569 std::array<double, Source::mem_size()> reference;
570 double sum = offset(node, target);
571 for (std::size_t source = 0; source < Source::mem_size(); ++source) {
572 reference[source] = input.mem(element, ddc::DiscreteElement<Source>(source));
573 sum += constant_response(node, target, source) * reference[source];
575 for (std::size_t entry
576 = target * StencilShape<Index, Radius>::size * Source::mem_size();
577 entry < (target + 1) * StencilShape<Index, Radius>::size * Source::mem_size();
579 std::size_t
const source = entry % Source::mem_size();
580 std::size_t
const point
581 = (entry / Source::mem_size()) % StencilShape<Index, Radius>::size;
582 double const weight = weights(node, entry);
588 Radius>(domain, element, point),
589 ddc::DiscreteElement<Source>(source))
590 - reference[source]);
592 output.mem(element, ddc::DiscreteElement<Target>(target)) = sum;
std::array< double, Target::mem_size()> offset
std::array< double, Target::mem_size() *Source::mem_size()> constant_response
Form(Coefficients const &) -> Form< Coefficients >
KOKKOS_FUNCTION constexpr auto tangent_basis(Elem elem)
std::array< double, Target::mem_size() *Source::mem_size() *detail::StencilShape< Index, Radius >::size > weights
KOKKOS_FUNCTION bool domain_contains(ddc::DiscreteDomain< DDim... > const &dom, ddc::DiscreteElement< ODDim... > elem)
KOKKOS_FUNCTION constexpr std::size_t factorial(std::size_t k) noexcept
Tensor(ddc::Chunk< ElementType, SupportType, Allocator >) -> Tensor< ElementType, SupportType, Kokkos::layout_right, typename Allocator::memory_space >
OwningTensor(TensorType, Storage &&) -> OwningTensor< TensorType, Storage >
Homogeneous bilinear coefficients, ordered output, first point/component, second point/component (sec...
static constexpr std::size_t first_size
std::array< double, Target::mem_size() *first_size *second_size > weights
static constexpr std::size_t second_size