7#include <similie/exterior/concepts.hpp>
8#include <similie/exterior/stencil.hpp>
9#include <similie/tensor/tensor_prod.hpp>
16template <
class Index,
class Tensor>
17KOKKOS_FUNCTION
auto local_component_tensor(Tensor
const& tensor)
19 static_assert(Tensor::discrete_domain_type::rank() <= 1);
20 using Span = ddc::ChunkSpan<
21 typename Tensor::element_type,
22 ddc::DiscreteDomain<Index>,
23 Kokkos::layout_stride,
24 typename Tensor::memory_space>;
25 std::size_t stride = 1;
26 std::size_t front = 0;
27 if constexpr (Tensor::discrete_domain_type::rank() == 1) {
28 stride = tensor.allocation_mdspan().stride(0);
29 front = ddc::detail::array(tensor.domain().front())[0];
32 Span(
typename Span::allocation_mdspan_type(
34 typename Span::mapping_type(
35 typename Span::extents_type(tensor.size()),
36 std::array<std::size_t, 1> {stride})),
37 ddc::DiscreteDomain<Index>(
38 ddc::DiscreteElement<Index>(front),
39 ddc::DiscreteVector<Index>(tensor.size()))));
45 template <
class...>
class Coefficients = tensor::TensorFullIndex,
49KOKKOS_FUNCTION
void apply_local_stencil(
52 Weights
const& weights,
55 using SourceComponent = tensor::Covariant<LocalComponentIndex<Source, false>>;
56 using TargetComponent = tensor::Covariant<LocalComponentIndex<Target, true>>;
57 using MatrixIndex = Coefficients<TargetComponent, tensor::upper_t<SourceComponent>>;
58 using MatrixSpan = ddc::ChunkSpan<
59 typename Output::element_type
const,
60 ddc::DiscreteDomain<MatrixIndex>,
61 Kokkos::layout_stride,
62 typename Output::memory_space>;
64 typename MatrixSpan::allocation_mdspan_type(
66 typename MatrixSpan::mapping_type(
67 typename MatrixSpan::extents_type(MatrixIndex::mem_size()),
68 std::array<std::size_t, 1> {weights.stride(1)})),
71 local_component_tensor<TargetComponent>(output),
73 local_component_tensor<SourceComponent>(input));
81 template <
class...>
class Coefficients = tensor::TensorDiagonalIndex,
84KOKKOS_FUNCTION
void apply_local_diagonal(
87 std::array<
double, LocalCoefficientIndex<Source, Target, Coefficients>::mem_size()>
const&
90 using SourceComponent = tensor::Covariant<LocalComponentIndex<Source, false>>;
91 using TargetComponent = tensor::Covariant<LocalComponentIndex<Target, true>>;
92 using Matrix = Coefficients<TargetComponent, tensor::upper_t<SourceComponent>>;
93 using Span = ddc::ChunkSpan<
95 ddc::DiscreteDomain<Matrix>,
96 Kokkos::layout_stride,
97 typename Output::memory_space>;
99 local_component_tensor<TargetComponent>(output),
101 Span(
typename Span::allocation_mdspan_type(
103 typename Span::mapping_type(
104 typename Span::extents_type(Matrix::mem_size()),
105 std::array<std::size_t, 1> {1})),
107 local_component_tensor<SourceComponent>(input));
111template <
class Source,
class Target,
template <
class...>
class Coefficients>
112KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> local_diagonal_stencil(
113 std::array<
double, LocalCoefficientIndex<Source, Target, Coefficients>::mem_size()>
const&
116 LocalOperatorStencil<Source, Target> result;
117 for (std::size_t component = 0; component < Source::mem_size(); ++component)
118 result.weights[component * Source::mem_size() + component]
119 = weights[LocalCoefficientIndex<Source, Target, Coefficients>::mem_id(
120 {component, component})];
125 template <
class...>
class Coefficients = tensor::TensorDiagonalIndex,
128KOKKOS_FUNCTION
void assert_local_diagonal(
129 [[maybe_unused]] LocalOperatorStencil<Source, Target>
const& stencil)
133 for (
double coefficient : stencil.weights)
134 scale = Kokkos::max(scale, Kokkos::abs(coefficient));
135 for (std::size_t row = 0; row < Target::mem_size(); ++row)
136 for (std::size_t col = 0; col < Source::mem_size(); ++col)
138 || Kokkos::abs(stencil.weights[row * Source::mem_size() + col])
139 <= 128. * std::numeric_limits<double>::epsilon() * scale)
140 &&
"Diagonal operator requires diagonal coefficients");
141 if constexpr (misc::Specialization<
142 LocalCoefficientIndex<Source, Target, Coefficients>,
143 tensor::TensorScalingIndex>) {
145 for (std::size_t component = 1; component < Source::mem_size(); ++component)
147 stencil.weights[component * Source::mem_size() + component]
148 - stencil.weights[0])
149 <= Kokkos::sqrt(std::numeric_limits<double>::epsilon()) * scale)
150 &&
"Scaling operator requires equal diagonal coefficients");
157template <
class Operator,
class...
Geometry>
158struct OperatorStencilValue;
160template <
class Operator>
161struct OperatorStencilValue<Operator>
163 template <
class Element,
class...
Geometry>
164 KOKKOS_FUNCTION
auto evaluate(Element element, Geometry
const&... geometry)
const
166 return Operator {}.value(geometry..., element);
170template <
class Operator,
class First,
class... Rest>
171struct OperatorStencilValue<Operator, First, Rest...>
174 OperatorStencilValue<Operator, Rest...> rest;
176 KOKKOS_FUNCTION OperatorStencilValue(First
const& first_, Rest
const&... rest_)
182 template <
class Element,
class... Bound>
183 KOKKOS_FUNCTION
auto evaluate(Element element, Bound
const&... bound)
const
185 return rest.evaluate(element, bound..., first);
188 template <
class Element>
189 KOKKOS_FUNCTION
auto operator()(Element element)
const
191 return evaluate(element);
203 class Exec =
typename Memory::execution_space,
204 template <
class...>
class Coefficients = tensor::TensorFullIndex,
205 class SourceElement =
typename tensor::natural_domain_t<Source>::discrete_element_type,
206 class TargetElement =
typename tensor::natural_domain_t<Target>::discrete_element_type,
207 bool NormalizeSource =
true>
208class LinearLocalOperatorStorage
210 using Element =
typename Domain::discrete_element_type;
211 LocalStencilStorage<Source, Target, Domain, Memory, Coefficients> m_stencils;
214 static LinearLocalOperatorStorage cache(
216 Domain
const& domain,
217 Geometry
const&... geometry)
219 LinearLocalOperatorStorage result;
220 result.m_stencils = cache_local_stencils<Source, Target, Memory, Coefficients>(
223 OperatorStencilValue<Operator, Geometry...>(geometry...));
228 using execution_space = Exec;
229 template <
class OtherExec,
template <
class...>
class OtherCoefficients = Coefficients>
230 using rebind = LinearLocalOperatorStorage<
242 template <
class Geometry>
243 requires requires(
Geometry const& geometry, Element element) {
244 Operator {}.value(geometry, element);
246 static LinearLocalOperatorStorage build(Exec
const& exec, Geometry
const& geometry)
248 return cache(exec, geometry.non_indices_domain(), geometry);
251 template <
class Geometry1,
class Geometry2>
252 requires requires(Geometry1
const& first, Geometry2
const&
second, Element element) {
253 Operator {}.value(first, second, element);
255 static LinearLocalOperatorStorage build(
257 Geometry1
const& first,
258 Geometry2
const& second)
260 return cache(exec, first.non_indices_domain(), first, second);
263 template <
class Geometry>
264 requires requires(
Geometry const& geometry, Element element) {
265 Operator {}.value(geometry, element);
267 static LinearLocalOperatorStorage build(
269 Geometry
const& geometry,
270 Domain
const& domain)
272 return cache(exec, domain, geometry);
275 template <
class Geometry1,
class Geometry2>
276 requires requires(Geometry1
const& first, Geometry2
const&
second, Element element) {
277 Operator {}.value(first, second, element);
279 static LinearLocalOperatorStorage build(
281 Geometry1
const& first,
282 Geometry2
const& second,
283 Domain
const& domain)
285 return cache(exec, domain, first, second);
288 template <ExteriorFieldType Output, ExteriorFieldType Input,
class...
Geometry>
290 sizeof...(Geometry) > 0
293 Element element) { Operator {}.value(geometry..., element); })
294 static LinearLocalOperatorStorage build(
296 Output
const& output,
298 Geometry
const&... geometry)
300 return cache(exec, output.non_indices_domain(), geometry...);
303 template <
class Output,
class Input>
304 KOKKOS_FUNCTION
void apply_pointwise(Output
const& output, Input
const& input, Element element)
307 apply_local_stencil<Source, Target, Coefficients>(
311 stencil_node(m_stencils.domain, element));
314 template <
class Output,
class Input,
class Geometry>
315 KOKKOS_FUNCTION
void apply_pointwise(
316 Output
const& output,
319 Element element)
const
321 apply_pointwise(output, input, element);
324 template <
class Output,
class Input,
class Geometry1,
class Geometry2>
325 KOKKOS_FUNCTION
void apply_pointwise(
326 Output
const& output,
330 Element element)
const
332 apply_pointwise(output, input, element);
335 template <ExteriorFieldType Output, ExteriorFieldType Input>
336 Output apply(Exec
const& exec, Output
const& output, Input
const& input)
const
338 LinearLocalOperatorStorage
const storage = *
this;
339 ddc::parallel_for_each(
340 "similie_apply_staged_local_operator",
342 output.non_indices_domain(),
343 KOKKOS_LAMBDA(Element element) {
344 storage.apply_pointwise(output[element], input[element], element);
349 KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> value(Element element)
const
351 LocalOperatorStencil<Source, Target> result;
352 using Index = LocalCoefficientIndex<Source, Target, Coefficients>;
353 std::size_t
const node = stencil_node(m_stencils.domain, element);
354 if constexpr (misc::Specialization<Index, tensor::TensorScalingIndex>) {
355 for (std::size_t component = 0; component < Source::mem_size(); ++component)
356 result.weights[component * Source::mem_size() + component]
357 = m_stencils.weights(node, 0);
359 for (std::size_t entry = 0; entry < Index::mem_size(); ++entry) {
360 std::array
const ids = Index::mem_id_to_canonical_natural_ids(entry);
361 result.weights[ids[0] * Source::mem_size() + ids[1]]
362 = m_stencils.weights(node, entry);
368 template <
class NaturalElement>
369 KOKKOS_FUNCTION
double value(Element element, NaturalElement natural)
const
371 return local_stencil_coefficient<Source, Target, NormalizeSource>(
373 ddc::detail::array(SourceElement(natural)),
374 ddc::detail::array(TargetElement(natural)));
377 template <
class Geometry>
378 KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> value(Geometry
const&, Element element)
381 return value(element);
384 template <
class Geometry,
class NaturalElement>
385 KOKKOS_FUNCTION
double value(Geometry
const&, Element element, NaturalElement natural)
const
387 return value(element, natural);
390 template <
class Geometry1,
class Geometry2>
391 KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> value(
394 Element element)
const
396 return value(element);
399 template <
class Geometry1,
class Geometry2,
class NaturalElement>
400 KOKKOS_FUNCTION
double value(
404 NaturalElement natural)
const
406 return value(element, natural);
416 class Exec =
typename Geometry::memory_space::execution_space>
417class ImplicitLocalOperatorStorage
419 using Element =
typename Geometry::non_indices_domain_t::discrete_element_type;
422 explicit ImplicitLocalOperatorStorage(Geometry
const& geometry) : m_geometry(geometry) {}
425 using execution_space = Exec;
426 template <
class OtherExec>
427 using rebind = ImplicitLocalOperatorStorage<Operator, Geometry, OtherExec>;
429 static ImplicitLocalOperatorStorage build(Exec
const&, Geometry
const& geometry)
431 return ImplicitLocalOperatorStorage(geometry);
434 template <
class Output,
class Input>
435 static ImplicitLocalOperatorStorage build(
439 Geometry
const& geometry)
441 return build(exec, geometry);
444 template <
class Output,
class Input>
445 KOKKOS_FUNCTION
void apply_pointwise(Output
const& output, Input
const& input, Element element)
448 Operator {}(output, input, m_geometry, element);
451 template <
class Output,
class Input>
452 KOKKOS_FUNCTION
void apply_pointwise(
453 Output
const& output,
456 Element element)
const
458 apply_pointwise(output, input, element);
461 template <ExteriorFieldType Output, ExteriorFieldType Input>
462 Output apply(Exec
const& exec, Output
const& output, Input
const& input)
const
464 ImplicitLocalOperatorStorage
const storage = *
this;
465 ddc::parallel_for_each(
466 "similie_apply_staged_implicit_operator",
468 output.non_indices_domain(),
469 KOKKOS_LAMBDA(Element element) {
470 storage.apply_pointwise(output[element], input[element], element);
475 KOKKOS_FUNCTION
auto value(Geometry, Element element)
const
477 return Operator {}.value(m_geometry, element);
480 template <
class NaturalElement>
481 KOKKOS_FUNCTION
double value(Geometry, Element element, NaturalElement natural)
const
483 return Operator {}.value(m_geometry, element, natural);
494 class Exec =
typename Memory::execution_space,
495 template <
class...>
class Coefficients = tensor::TensorFullIndex,
496 class SourceElement =
typename tensor::natural_domain_t<Source>::discrete_element_type,
497 class TargetElement =
typename tensor::natural_domain_t<Target>::discrete_element_type,
498 bool NormalizeSource =
true>
499using LocalOperatorStorage = LinearLocalOperatorStorage<
static KOKKOS_FUNCTION constexpr discrete_domain_type domain()
Tensor(ddc::Chunk< ElementType, SupportType, Allocator >) -> Tensor< ElementType, SupportType, Kokkos::layout_right, typename Allocator::memory_space >
Tensor< ElementType, ddc::DiscreteDomain< ProdDDim... >, LayoutStridedPolicy, MemorySpace > tensor_prod(Tensor< ElementType, ddc::DiscreteDomain< ProdDDim... >, LayoutStridedPolicy, MemorySpace > const &prod_tensor, Tensor< ElementType, ddc::DiscreteDomain< DDim1... >, LayoutStridedPolicy, MemorySpace > const &tensor1, Tensor< ElementType, ddc::DiscreteDomain< DDim2... >, LayoutStridedPolicy, MemorySpace > const &tensor2)