SimiLie
Loading...
Searching...
No Matches
local_operator_storage.hpp
1// SPDX-FileCopyrightText: 2026 Baptiste Legouix
2// SPDX-License-Identifier: AGPL-3.0-or-later
3// AI-GENERATED
4
5#pragma once
6
7#include <similie/exterior/concepts.hpp>
8#include <similie/exterior/stencil.hpp>
9#include <similie/tensor/tensor_prod.hpp>
10
11namespace sil::exterior {
12namespace detail {
13// Canonical cochain components are an ordinary vector space: the wedge signs
14// and factorial factors are already included in the cached coefficient matrix.
15// Retag the existing storage, retaining its stride, rather than expanding forms.
16template <class Index, class Tensor>
17KOKKOS_FUNCTION auto local_component_tensor(Tensor const& tensor)
18{
19 static_assert(Tensor::discrete_domain_type::rank() <= 1);
20 using Span = ddc::ChunkSpan<
21 typename Tensor::element_type,
22 ddc::DiscreteDomain<Index>,
23 Kokkos::layout_stride,
24 typename Tensor::memory_space>;
25 std::size_t stride = 1;
26 std::size_t front = 0;
27 if constexpr (Tensor::discrete_domain_type::rank() == 1) {
28 stride = tensor.allocation_mdspan().stride(0);
29 front = ddc::detail::array(tensor.domain().front())[0];
30 }
31 return tensor::Tensor(
32 Span(typename Span::allocation_mdspan_type(
33 tensor.data_handle(),
34 typename Span::mapping_type(
35 typename Span::extents_type(tensor.size()),
36 std::array<std::size_t, 1> {stride})),
37 ddc::DiscreteDomain<Index>(
38 ddc::DiscreteElement<Index>(front),
39 ddc::DiscreteVector<Index>(tensor.size()))));
40}
41
42template <
43 class Source,
44 class Target,
45 template <class...> class Coefficients = tensor::TensorFullIndex,
46 class Output,
47 class Input,
48 class Weights>
49KOKKOS_FUNCTION void apply_local_stencil(
50 Output const& output,
51 Input const& input,
52 Weights const& weights,
53 std::size_t node)
54{
55 using SourceComponent = tensor::Covariant<LocalComponentIndex<Source, false>>;
56 using TargetComponent = tensor::Covariant<LocalComponentIndex<Target, true>>;
57 using MatrixIndex = Coefficients<TargetComponent, tensor::upper_t<SourceComponent>>;
58 using MatrixSpan = ddc::ChunkSpan<
59 typename Output::element_type const,
60 ddc::DiscreteDomain<MatrixIndex>,
61 Kokkos::layout_stride,
62 typename Output::memory_space>;
63 tensor::Tensor matrix(MatrixSpan(
64 typename MatrixSpan::allocation_mdspan_type(
65 &weights(node, 0),
66 typename MatrixSpan::mapping_type(
67 typename MatrixSpan::extents_type(MatrixIndex::mem_size()),
68 std::array<std::size_t, 1> {weights.stride(1)})),
71 local_component_tensor<TargetComponent>(output),
72 matrix,
73 local_component_tensor<SourceComponent>(input));
74}
75
76// Apply coefficients computed at the current point through the same diagonal
77// tensor product as a cached local operator, without allocating a cache.
78template <
79 class Source,
80 class Target,
81 template <class...> class Coefficients = tensor::TensorDiagonalIndex,
82 class Output,
83 class Input>
84KOKKOS_FUNCTION void apply_local_diagonal(
85 Output const& output,
86 Input const& input,
87 std::array<double, LocalCoefficientIndex<Source, Target, Coefficients>::mem_size()> const&
88 weights)
89{
90 using SourceComponent = tensor::Covariant<LocalComponentIndex<Source, false>>;
91 using TargetComponent = tensor::Covariant<LocalComponentIndex<Target, true>>;
92 using Matrix = Coefficients<TargetComponent, tensor::upper_t<SourceComponent>>;
93 using Span = ddc::ChunkSpan<
94 double const,
95 ddc::DiscreteDomain<Matrix>,
96 Kokkos::layout_stride,
97 typename Output::memory_space>;
99 local_component_tensor<TargetComponent>(output),
101 Span(typename Span::allocation_mdspan_type(
102 weights.data(),
103 typename Span::mapping_type(
104 typename Span::extents_type(Matrix::mem_size()),
105 std::array<std::size_t, 1> {1})),
107 local_component_tensor<SourceComponent>(input));
108}
109
110// Expand compressed diagonal coefficients only when a caller requests a full stencil.
111template <class Source, class Target, template <class...> class Coefficients>
112KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> local_diagonal_stencil(
113 std::array<double, LocalCoefficientIndex<Source, Target, Coefficients>::mem_size()> const&
114 weights)
115{
116 LocalOperatorStencil<Source, Target> result;
117 for (std::size_t component = 0; component < Source::mem_size(); ++component)
118 result.weights[component * Source::mem_size() + component]
119 = weights[LocalCoefficientIndex<Source, Target, Coefficients>::mem_id(
120 {component, component})];
121 return result;
122}
123
124template <
125 template <class...> class Coefficients = tensor::TensorDiagonalIndex,
126 class Source,
127 class Target>
128KOKKOS_FUNCTION void assert_local_diagonal(
129 [[maybe_unused]] LocalOperatorStencil<Source, Target> const& stencil)
130{
131#ifndef NDEBUG
132 double scale = 0.;
133 for (double coefficient : stencil.weights)
134 scale = Kokkos::max(scale, Kokkos::abs(coefficient));
135 for (std::size_t row = 0; row < Target::mem_size(); ++row)
136 for (std::size_t col = 0; col < Source::mem_size(); ++col)
137 assert((row == col
138 || Kokkos::abs(stencil.weights[row * Source::mem_size() + col])
139 <= 128. * std::numeric_limits<double>::epsilon() * scale)
140 && "Diagonal operator requires diagonal coefficients");
141 if constexpr (misc::Specialization<
142 LocalCoefficientIndex<Source, Target, Coefficients>,
143 tensor::TensorScalingIndex>) {
144 // Edge lengths come from coordinate differences, which can amplify roundoff.
145 for (std::size_t component = 1; component < Source::mem_size(); ++component)
146 assert((Kokkos::abs(
147 stencil.weights[component * Source::mem_size() + component]
148 - stencil.weights[0])
149 <= Kokkos::sqrt(std::numeric_limits<double>::epsilon()) * scale)
150 && "Scaling operator requires equal diagonal coefficients");
151 }
152#endif
153}
154
155
156// Bind fixed geometric arguments; the evaluation element remains dynamic.
157template <class Operator, class... Geometry>
158struct OperatorStencilValue;
159
160template <class Operator>
161struct OperatorStencilValue<Operator>
162{
163 template <class Element, class... Geometry>
164 KOKKOS_FUNCTION auto evaluate(Element element, Geometry const&... geometry) const
165 {
166 return Operator {}.value(geometry..., element);
167 }
168};
169
170template <class Operator, class First, class... Rest>
171struct OperatorStencilValue<Operator, First, Rest...>
172{
173 First first;
174 OperatorStencilValue<Operator, Rest...> rest;
175
176 KOKKOS_FUNCTION OperatorStencilValue(First const& first_, Rest const&... rest_)
177 : first(first_)
178 , rest(rest_...)
179 {
180 }
181
182 template <class Element, class... Bound>
183 KOKKOS_FUNCTION auto evaluate(Element element, Bound const&... bound) const
184 {
185 return rest.evaluate(element, bound..., first);
186 }
187
188 template <class Element>
189 KOKKOS_FUNCTION auto operator()(Element element) const
190 {
191 return evaluate(element);
192 }
193};
194
195// The policy retains the native coefficient index: dense and diagonal maps do
196// not acquire a common expanded representation during either build or apply.
197template <
198 class Operator,
199 class Source,
200 class Target,
201 class Domain,
202 class Memory,
203 class Exec = typename Memory::execution_space,
204 template <class...> class Coefficients = tensor::TensorFullIndex,
205 class SourceElement = typename tensor::natural_domain_t<Source>::discrete_element_type,
206 class TargetElement = typename tensor::natural_domain_t<Target>::discrete_element_type,
207 bool NormalizeSource = true>
208class LinearLocalOperatorStorage
209{
210 using Element = typename Domain::discrete_element_type;
211 LocalStencilStorage<Source, Target, Domain, Memory, Coefficients> m_stencils;
212
213 template <class... Geometry>
214 static LinearLocalOperatorStorage cache(
215 Exec const& exec,
216 Domain const& domain,
217 Geometry const&... geometry)
218 {
219 LinearLocalOperatorStorage result;
220 result.m_stencils = cache_local_stencils<Source, Target, Memory, Coefficients>(
221 exec,
222 domain,
223 OperatorStencilValue<Operator, Geometry...>(geometry...));
224 return result;
225 }
226
227public:
228 using execution_space = Exec;
229 template <class OtherExec, template <class...> class OtherCoefficients = Coefficients>
230 using rebind = LinearLocalOperatorStorage<
231 Operator,
232 Source,
233 Target,
234 Domain,
235 Memory,
236 OtherExec,
237 OtherCoefficients,
238 SourceElement,
239 TargetElement,
240 NormalizeSource>;
241
242 template <class Geometry>
243 requires requires(Geometry const& geometry, Element element) {
244 Operator {}.value(geometry, element);
245 }
246 static LinearLocalOperatorStorage build(Exec const& exec, Geometry const& geometry)
247 {
248 return cache(exec, geometry.non_indices_domain(), geometry);
249 }
250
251 template <class Geometry1, class Geometry2>
252 requires requires(Geometry1 const& first, Geometry2 const& second, Element element) {
253 Operator {}.value(first, second, element);
254 }
255 static LinearLocalOperatorStorage build(
256 Exec const& exec,
257 Geometry1 const& first,
258 Geometry2 const& second)
259 {
260 return cache(exec, first.non_indices_domain(), first, second);
261 }
262
263 template <class Geometry>
264 requires requires(Geometry const& geometry, Element element) {
265 Operator {}.value(geometry, element);
266 }
267 static LinearLocalOperatorStorage build(
268 Exec const& exec,
269 Geometry const& geometry,
270 Domain const& domain)
271 {
272 return cache(exec, domain, geometry);
273 }
274
275 template <class Geometry1, class Geometry2>
276 requires requires(Geometry1 const& first, Geometry2 const& second, Element element) {
277 Operator {}.value(first, second, element);
278 }
279 static LinearLocalOperatorStorage build(
280 Exec const& exec,
281 Geometry1 const& first,
282 Geometry2 const& second,
283 Domain const& domain)
284 {
285 return cache(exec, domain, first, second);
286 }
287
288 template <ExteriorFieldType Output, ExteriorFieldType Input, class... Geometry>
289 requires(
290 sizeof...(Geometry) > 0
291 && requires(
292 Geometry const&... geometry,
293 Element element) { Operator {}.value(geometry..., element); })
294 static LinearLocalOperatorStorage build(
295 Exec const& exec,
296 Output const& output,
297 Input const&,
298 Geometry const&... geometry)
299 {
300 return cache(exec, output.non_indices_domain(), geometry...);
301 }
302
303 template <class Output, class Input>
304 KOKKOS_FUNCTION void apply_pointwise(Output const& output, Input const& input, Element element)
305 const
306 {
307 apply_local_stencil<Source, Target, Coefficients>(
308 output,
309 input,
310 m_stencils.weights,
311 stencil_node(m_stencils.domain, element));
312 }
313
314 template <class Output, class Input, class Geometry>
315 KOKKOS_FUNCTION void apply_pointwise(
316 Output const& output,
317 Input const& input,
318 Geometry const&,
319 Element element) const
320 {
321 apply_pointwise(output, input, element);
322 }
323
324 template <class Output, class Input, class Geometry1, class Geometry2>
325 KOKKOS_FUNCTION void apply_pointwise(
326 Output const& output,
327 Input const& input,
328 Geometry1 const&,
329 Geometry2 const&,
330 Element element) const
331 {
332 apply_pointwise(output, input, element);
333 }
334
335 template <ExteriorFieldType Output, ExteriorFieldType Input>
336 Output apply(Exec const& exec, Output const& output, Input const& input) const
337 {
338 LinearLocalOperatorStorage const storage = *this;
339 ddc::parallel_for_each(
340 "similie_apply_staged_local_operator",
341 exec,
342 output.non_indices_domain(),
343 KOKKOS_LAMBDA(Element element) {
344 storage.apply_pointwise(output[element], input[element], element);
345 });
346 return output;
347 }
348
349 KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> value(Element element) const
350 {
351 LocalOperatorStencil<Source, Target> result;
352 using Index = LocalCoefficientIndex<Source, Target, Coefficients>;
353 std::size_t const node = stencil_node(m_stencils.domain, element);
354 if constexpr (misc::Specialization<Index, tensor::TensorScalingIndex>) {
355 for (std::size_t component = 0; component < Source::mem_size(); ++component)
356 result.weights[component * Source::mem_size() + component]
357 = m_stencils.weights(node, 0);
358 } else {
359 for (std::size_t entry = 0; entry < Index::mem_size(); ++entry) {
360 std::array const ids = Index::mem_id_to_canonical_natural_ids(entry);
361 result.weights[ids[0] * Source::mem_size() + ids[1]]
362 = m_stencils.weights(node, entry);
363 }
364 }
365 return result;
366 }
367
368 template <class NaturalElement>
369 KOKKOS_FUNCTION double value(Element element, NaturalElement natural) const
370 {
371 return local_stencil_coefficient<Source, Target, NormalizeSource>(
372 value(element),
373 ddc::detail::array(SourceElement(natural)),
374 ddc::detail::array(TargetElement(natural)));
375 }
376
377 template <class Geometry>
378 KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> value(Geometry const&, Element element)
379 const
380 {
381 return value(element);
382 }
383
384 template <class Geometry, class NaturalElement>
385 KOKKOS_FUNCTION double value(Geometry const&, Element element, NaturalElement natural) const
386 {
387 return value(element, natural);
388 }
389
390 template <class Geometry1, class Geometry2>
391 KOKKOS_FUNCTION LocalOperatorStencil<Source, Target> value(
392 Geometry1 const&,
393 Geometry2 const&,
394 Element element) const
395 {
396 return value(element);
397 }
398
399 template <class Geometry1, class Geometry2, class NaturalElement>
400 KOKKOS_FUNCTION double value(
401 Geometry1 const&,
402 Geometry2 const&,
403 Element element,
404 NaturalElement natural) const
405 {
406 return value(element, natural);
407 }
408};
409
410// Use only when the operator type proves that its coefficients do not depend
411// on geometry. The retained descriptor supplies domain/type information, not
412// a per-node coefficient allocation.
413template <
414 class Operator,
415 class Geometry,
416 class Exec = typename Geometry::memory_space::execution_space>
417class ImplicitLocalOperatorStorage
418{
419 using Element = typename Geometry::non_indices_domain_t::discrete_element_type;
420 Geometry m_geometry;
421
422 explicit ImplicitLocalOperatorStorage(Geometry const& geometry) : m_geometry(geometry) {}
423
424public:
425 using execution_space = Exec;
426 template <class OtherExec>
427 using rebind = ImplicitLocalOperatorStorage<Operator, Geometry, OtherExec>;
428
429 static ImplicitLocalOperatorStorage build(Exec const&, Geometry const& geometry)
430 {
431 return ImplicitLocalOperatorStorage(geometry);
432 }
433
434 template <class Output, class Input>
435 static ImplicitLocalOperatorStorage build(
436 Exec const& exec,
437 Output const&,
438 Input const&,
439 Geometry const& geometry)
440 {
441 return build(exec, geometry);
442 }
443
444 template <class Output, class Input>
445 KOKKOS_FUNCTION void apply_pointwise(Output const& output, Input const& input, Element element)
446 const
447 {
448 Operator {}(output, input, m_geometry, element);
449 }
450
451 template <class Output, class Input>
452 KOKKOS_FUNCTION void apply_pointwise(
453 Output const& output,
454 Input const& input,
455 Geometry const&,
456 Element element) const
457 {
458 apply_pointwise(output, input, element);
459 }
460
461 template <ExteriorFieldType Output, ExteriorFieldType Input>
462 Output apply(Exec const& exec, Output const& output, Input const& input) const
463 {
464 ImplicitLocalOperatorStorage const storage = *this;
465 ddc::parallel_for_each(
466 "similie_apply_staged_implicit_operator",
467 exec,
468 output.non_indices_domain(),
469 KOKKOS_LAMBDA(Element element) {
470 storage.apply_pointwise(output[element], input[element], element);
471 });
472 return output;
473 }
474
475 KOKKOS_FUNCTION auto value(Geometry, Element element) const
476 {
477 return Operator {}.value(m_geometry, element);
478 }
479
480 template <class NaturalElement>
481 KOKKOS_FUNCTION double value(Geometry, Element element, NaturalElement natural) const
482 {
483 return Operator {}.value(m_geometry, element, natural);
484 }
485};
486
487// Compatibility spelling for existing linear operators.
488template <
489 class Operator,
490 class Source,
491 class Target,
492 class Domain,
493 class Memory,
494 class Exec = typename Memory::execution_space,
495 template <class...> class Coefficients = tensor::TensorFullIndex,
496 class SourceElement = typename tensor::natural_domain_t<Source>::discrete_element_type,
497 class TargetElement = typename tensor::natural_domain_t<Target>::discrete_element_type,
498 bool NormalizeSource = true>
499using LocalOperatorStorage = LinearLocalOperatorStorage<
500 Operator,
501 Source,
502 Target,
503 Domain,
504 Memory,
505 Exec,
506 Coefficients,
507 SourceElement,
508 TargetElement,
509 NormalizeSource>;
510
511} // namespace detail
512} // namespace sil::exterior
static KOKKOS_FUNCTION constexpr discrete_domain_type domain()
Tensor(ddc::Chunk< ElementType, SupportType, Allocator >) -> Tensor< ElementType, SupportType, Kokkos::layout_right, typename Allocator::memory_space >
prime< Index, 2 > second
Definition prime.hpp:21
Tensor< ElementType, ddc::DiscreteDomain< ProdDDim... >, LayoutStridedPolicy, MemorySpace > tensor_prod(Tensor< ElementType, ddc::DiscreteDomain< ProdDDim... >, LayoutStridedPolicy, MemorySpace > const &prod_tensor, Tensor< ElementType, ddc::DiscreteDomain< DDim1... >, LayoutStridedPolicy, MemorySpace > const &tensor1, Tensor< ElementType, ddc::DiscreteDomain< DDim2... >, LayoutStridedPolicy, MemorySpace > const &tensor2)