SimiLie
Loading...
Searching...
No Matches
stencil.hpp
1// SPDX-FileCopyrightText: 2026 Baptiste Legouix
2// SPDX-License-Identifier: AGPL-3.0-or-later
3// AI-GENERATED
4
5#pragma once
6
7#include <array>
8#include <cassert>
9#include <cstddef>
10#include <limits>
11#include <stdexcept>
12#include <type_traits>
13#include <utility>
14
15#include <similie/exterior/coboundary/coboundary.hpp>
16#include <similie/tensor/tensor_indices/diagonal_tensor.hpp>
17#include <similie/tensor/tensor_indices/scaling_tensor.hpp>
18
19#include "stencil_indices.hpp"
20
21namespace sil::exterior {
22
23// Full coefficient layout: output component, spatial point, input component.
24// Offsets and constant responses support affine closures and constant null modes.
25template <class Index, class Source, class Target, std::size_t Radius>
26struct Stencil
27{
28 std::array<
29 double,
30 Target::mem_size() * Source::mem_size() * detail::StencilShape<Index, Radius>::size>
32 std::array<double, Target::mem_size()> offset {};
33 std::array<double, Target::mem_size() * Source::mem_size()> constant_response {};
34 bool complete = true;
35};
36
37// Historical Stencil includes affine boundary responses; its dynamic action is linear.
38template <class Index, class Source, class Target, std::size_t Radius>
40
47template <
48 class Index,
49 class First,
50 class Second,
51 class Target,
52 std::size_t FirstRadius,
53 std::size_t SecondRadius>
55{
56 static constexpr std::size_t first_size
57 = detail::StencilShape<Index, FirstRadius>::size * First::mem_size();
58 static constexpr std::size_t second_size
59 = detail::StencilShape<Index, SecondRadius>::size * Second::mem_size();
60 std::array<double, Target::mem_size() * first_size * second_size> weights {};
61 bool complete = true;
62};
63
64namespace detail {
65
66// Static tensor coefficients, without spatial dimensions or heap allocation.
67template <class Memory, class... Indices>
68KOKKOS_FUNCTION auto coefficient_stencil(ddc::DiscreteDomain<Indices...> const& domain)
69{
70 std::array<double, (Indices::mem_size() * ... * std::size_t(1))> values {};
71 tensor::Tensor<double, ddc::DiscreteDomain<Indices...>, Kokkos::layout_right, Memory> view(
72 ddc::ChunkSpan<
73 double,
74 ddc::DiscreteDomain<Indices...>,
75 Kokkos::layout_right,
76 Memory>(values.data(), domain));
77 return tensor::OwningTensor(view, std::move(values));
78}
79
80// A local operator uses the same ordering with one spatial point.
81template <class Source, class Target>
82struct LocalOperatorStencil
83{
84 std::array<double, Source::mem_size() * Target::mem_size()> weights {};
85};
86
87// Recover a natural tensor coefficient from canonical cochain components.
88// Antisymmetric permutations and factorial conventions are applied only here.
89template <
90 class Source,
91 class Target,
92 bool NormalizeSource = true,
93 class Stencil,
94 std::size_t SourceRank,
95 std::size_t TargetRank>
96KOKKOS_FUNCTION double local_stencil_coefficient(
97 Stencil const& stencil,
98 std::array<std::size_t, SourceRank> source,
99 std::array<std::size_t, TargetRank> target)
100{
101 int sign = 1;
102 for (std::size_t i = 0; i < source.size(); ++i)
103 for (std::size_t j = i + 1; j < source.size(); ++j) {
104 if (source[i] == source[j])
105 return 0.0;
106 if (source[i] > source[j]) {
107 Kokkos::kokkos_swap(source[i], source[j]);
108 sign = -sign;
109 }
110 }
111 for (std::size_t i = 0; i < target.size(); ++i)
112 for (std::size_t j = i + 1; j < target.size(); ++j) {
113 if (target[i] == target[j])
114 return 0.0;
115 if (target[i] > target[j]) {
116 Kokkos::kokkos_swap(target[i], target[j]);
117 sign = -sign;
118 }
119 }
120 std::size_t factorial = 1;
121 if constexpr (NormalizeSource)
122 for (std::size_t i = 2; i <= source.size(); ++i)
123 factorial *= i;
124 return sign
125 * stencil.weights
126 [stencil_component_index<Target>(target) * Source::mem_size()
127 + stencil_component_index<Source>(source)]
128 / factorial;
129}
130
131// Stored topology is shared by pointwise compositions, not copied down their call tree.
132template <std::size_t Rank, class Domain, class Memory>
133class StencilChain
134{
135 using Basis = decltype(tangent_basis<Rank, Domain, Memory>(
136 typename Domain::discrete_element_type {}));
137 Kokkos::View<Basis, Memory> m_basis;
138
139public:
140 using simplex_type = typename Basis::simplex_type;
141 using memory_space = Memory;
142 StencilChain() : m_basis(Kokkos::view_alloc(Kokkos::WithoutInitializing, "stencil_topology"))
143 {
144 Basis const basis
145 = tangent_basis<Rank, Domain, Memory>(typename Domain::discrete_element_type {});
146 Kokkos::deep_copy(m_basis, basis);
147 }
148 KOKKOS_FORCEINLINE_FUNCTION std::size_t size() const
149 {
150 return m_basis().size();
151 }
152 KOKKOS_FORCEINLINE_FUNCTION simplex_type const& operator[](std::size_t i) const
153 {
154 return m_basis()[i];
155 }
156 KOKKOS_FORCEINLINE_FUNCTION auto begin() const
157 {
158 return m_basis().begin();
159 }
160 KOKKOS_FORCEINLINE_FUNCTION auto end() const
161 {
162 return m_basis().end();
163 }
164};
165
166template <std::size_t Rank, class Domain, class Memory>
167struct StencilChains
168{
169 StencilChain<Rank, Domain, Memory> chain;
170 std::conditional_t<
171 (Rank < Domain::rank()),
172 StencilChains<Rank + 1, Domain, Memory>,
173 std::nullptr_t>
174 tail;
175 template <std::size_t Selected>
176 KOKKOS_FORCEINLINE_FUNCTION auto get() const
177 {
178 if constexpr (Selected == Rank)
179 return chain;
180 else
181 return tail.template get<Selected>();
182 }
183};
184
185template <class Domain, class Element>
186KOKKOS_FORCEINLINE_FUNCTION std::size_t stencil_node(Domain const& domain, Element element)
187{
188 std::size_t node = 0;
189 for (std::size_t axis = 0; axis < Domain::rank(); ++axis)
190 node = node * ddc::detail::array(domain.extents())[axis] + ddc::detail::array(element)[axis]
191 - ddc::detail::array(domain.front())[axis];
192 return node;
193}
194
195template <class Domain>
196KOKKOS_FORCEINLINE_FUNCTION typename Domain::discrete_element_type stencil_element(
197 Domain const& domain,
198 std::size_t node)
199{
200 typename Domain::discrete_element_type element = domain.front();
201 for (std::size_t axis = Domain::rank(); axis-- > 0;) {
202 ddc::detail::array(element)[axis] += node % ddc::detail::array(domain.extents())[axis];
203 node /= ddc::detail::array(domain.extents())[axis];
204 }
205 return element;
206}
207
208template <class Index, class... Dimensions>
209constexpr std::array<bool, sizeof...(Dimensions)> stencil_axes(ddc::DiscreteDomain<Dimensions...>)
210{
211 return {ddc::type_seq_contains_v<
212 ddc::TypeSeq<typename Dimensions::continuous_dimension_type>,
213 typename Index::type_seq_dimensions>...};
214}
215
216// Cyclic addressing supplies an implicit numbering at periodic boundaries.
217// Nonperiodic closures simply give zero coefficients to unused wrapped samples.
218template <class Index, std::size_t Radius, class Domain, class Element>
219KOKKOS_FORCEINLINE_FUNCTION Element
220stencil_source(Domain const& domain, Element element, std::size_t point)
221{
222 auto const offset = StencilShape<Index, Radius>::offset(point);
223 auto const active = stencil_axes<Index>(Domain {});
224 std::size_t physical_axis = 0;
225 for (std::size_t axis = 0; axis < Domain::rank(); ++axis) {
226 if (!active[axis])
227 continue;
228 std::ptrdiff_t const front = ddc::detail::array(domain.front())[axis];
229 std::ptrdiff_t const count = ddc::detail::array(domain.extents())[axis];
230 std::ptrdiff_t position = std::ptrdiff_t(ddc::detail::array(element)[axis]) - front
231 + offset[physical_axis++];
232 if (position < 0 || position >= count)
233 position = (position % count + count) % count;
234 ddc::detail::array(element)[axis] = front + position;
235 }
236 return element;
237}
238
239template <
240 class Index,
241 class Source,
242 class Target,
243 std::size_t Radius,
244 class Domain,
245 class Element,
246 class Evaluate>
247KOKKOS_FORCEINLINE_FUNCTION Stencil<Index, Source, Target, Radius> operator_stencil(
248 Domain const& domain,
249 Element element,
250 Evaluate const& evaluate)
251{
252 Stencil<Index, Source, Target, Radius> stencil;
253 auto baseline = evaluate([&](auto sample, auto) {
254 bool found = false;
255 for (std::size_t point = 0; point < StencilShape<Index, Radius>::size; ++point)
256 found |= stencil_source<Index, Radius>(domain, element, point) == sample;
257 stencil.complete &= found;
258 return 0.0;
259 });
260 for (std::size_t target = 0; target < Target::mem_size(); ++target)
261 stencil.offset[target] = baseline.mem(ddc::DiscreteElement<Target>(target));
262 for (std::size_t source = 0; source < Source::mem_size(); ++source) {
263 auto response = evaluate([&](auto, auto component) {
264 return component.template uid<Source>() == source ? 1.0 : 0.0;
265 });
266 for (std::size_t target = 0; target < Target::mem_size(); ++target)
267 stencil.constant_response[target * Source::mem_size() + source]
268 = response.mem(ddc::DiscreteElement<Target>(target)) - stencil.offset[target];
269 }
270 for (std::size_t point = 0; point < StencilShape<Index, Radius>::size; ++point) {
271 Element const sample = stencil_source<Index, Radius>(domain, element, point);
272 bool duplicate = false;
273 for (std::size_t earlier = 0; earlier < point; ++earlier)
274 duplicate |= stencil_source<Index, Radius>(domain, element, earlier) == sample;
275 if (duplicate)
276 continue;
277 for (std::size_t source = 0; source < Source::mem_size(); ++source) {
278 auto response = evaluate([&](auto sampled_element, auto component) {
279 return sampled_element == sample && component.template uid<Source>() == source
280 ? 1.0
281 : 0.0;
282 });
283 for (std::size_t target = 0; target < Target::mem_size(); ++target)
284 stencil.weights
285 [(target * StencilShape<Index, Radius>::size + point) * Source::mem_size()
286 + source]
287 = response.mem(ddc::DiscreteElement<Target>(target))
288 - stencil.offset[target];
289 }
290 }
291 return stencil;
292}
293
294// A lazy field lets the fundamental operator evaluate unit inputs without an allocation.
295template <class Index, class Domain, class Memory, class Evaluate>
296class StencilField
297{
298 Domain m_domain;
299 Evaluate m_evaluate;
300
301public:
302 using memory_space = Memory;
303 using non_indices_domain_t = Domain;
304 using indices_domain_t = ddc::DiscreteDomain<Index>;
305 KOKKOS_FORCEINLINE_FUNCTION StencilField(Domain const& domain, Evaluate const& evaluate)
306 : m_domain(domain)
307 , m_evaluate(evaluate)
308 {
309 }
310 KOKKOS_FORCEINLINE_FUNCTION Domain non_indices_domain() const
311 {
312 return m_domain;
313 }
314 KOKKOS_FORCEINLINE_FUNCTION static tensor::TensorAccessor<Index> accessor()
315 {
316 return {};
317 }
318 template <class Element, class Component>
319 KOKKOS_FORCEINLINE_FUNCTION double mem(Element element, Component component) const
320 {
321 return m_evaluate(element, component);
322 }
323 template <class Element>
324 KOKKOS_FORCEINLINE_FUNCTION auto operator[](Element element) const
325 {
326 return tensor::detail::pointwise_tensor<indices_domain_t, Memory>(
327 [&](auto component) { return mem(element, component); });
328 }
329};
330
331template <class Index, class Source, class Target, std::size_t Radius, class Domain, class Memory>
332struct StencilStorage
333{
334 Domain domain;
335 Kokkos::View<
336 double * [Target::mem_size() * StencilShape<Index, Radius>::size * Source::mem_size()],
337 Kokkos::LayoutLeft,
338 Memory>
339 weights;
340 Kokkos::View<double * [Target::mem_size()], Kokkos::LayoutLeft, Memory> offset;
341 Kokkos::View<double * [Target::mem_size()][Source::mem_size()], Kokkos::LayoutLeft, Memory>
342 constant_response;
343};
344
345// Matrix indices enumerate independent form components, not physical directions.
346template <std::size_t Component>
347struct LocalComponent
348{
349};
350
351template <class Form, bool Row, class Sequence = std::make_index_sequence<Form::mem_size()>>
352struct LocalComponentIndex;
353
354template <class Form, bool Row, std::size_t... Components>
355struct LocalComponentIndex<Form, Row, std::index_sequence<Components...>>
356 : tensor::TensorNaturalIndex<LocalComponent<Components>...>
357{
358};
359
360template <class Index>
361concept LocalDiagonalCoefficientIndex = misc::Specialization<Index, tensor::TensorDiagonalIndex>
362 || misc::Specialization<Index, tensor::TensorScalingIndex>;
363
364template <class Source, class Target, template <class...> class Coefficients>
365using LocalCoefficientIndex
366 = Coefficients<LocalComponentIndex<Target, true>, LocalComponentIndex<Source, false>>;
367
368template <
369 class Source,
370 class Target,
371 class Domain,
372 class Memory,
373 template <class...> class Coefficients = tensor::TensorFullIndex>
374struct LocalStencilStorage
375{
376 Domain domain;
377 Kokkos::View<
378 double * [LocalCoefficientIndex<Source, Target, Coefficients>::mem_size()],
379 Kokkos::LayoutLeft,
380 Memory>
381 weights;
382};
383
384template <
385 class Source,
386 class Target,
387 class Memory,
388 template <class...> class Coefficients = tensor::TensorFullIndex,
389 class Exec,
390 class Domain,
391 class Evaluate>
392LocalStencilStorage<Source, Target, Domain, Memory, Coefficients> cache_local_stencils(
393 Exec const& exec,
394 Domain domain,
395 Evaluate const& evaluate)
396{
397 using Index = LocalCoefficientIndex<Source, Target, Coefficients>;
398 static_assert(
399 misc::Specialization<Index, tensor::TensorFullIndex>
400 || misc::Specialization<Index, tensor::TensorDiagonalIndex>
401 || misc::Specialization<Index, tensor::TensorScalingIndex>);
402 if constexpr (LocalDiagonalCoefficientIndex<Index>)
403 static_assert(Source::mem_size() == Target::mem_size());
404 LocalStencilStorage<Source, Target, Domain, Memory, Coefficients> storage;
405 storage.domain = domain;
406 storage.weights = decltype(storage.weights)(
407 Kokkos::view_alloc(Kokkos::WithoutInitializing, "staged_local_stencils"),
408 domain.size());
409 auto const weights = storage.weights;
410 Kokkos::parallel_for(
411 "similie_cache_local_stencils",
412 Kokkos::RangePolicy<Exec>(exec, 0, domain.size()),
413 KOKKOS_LAMBDA(std::size_t node) {
414 auto const stencil = evaluate(stencil_element(domain, node));
415#ifndef NDEBUG
416 if constexpr (LocalDiagonalCoefficientIndex<Index>) {
417 double scale = 0.0;
418 for (double coefficient : stencil.weights)
419 scale = Kokkos::max(scale, Kokkos::abs(coefficient));
420 for (std::size_t row = 0; row < Target::mem_size(); ++row)
421 for (std::size_t col = 0; col < Source::mem_size(); ++col)
422 assert((row == col
423 || Kokkos::abs(stencil.weights[row * Source::mem_size() + col])
424 <= 128.0 * std::numeric_limits<double>::epsilon()
425 * scale)
426 && "Diagonal staged operator requires diagonal coefficients");
427 if constexpr (misc::Specialization<Index, tensor::TensorScalingIndex>) {
428 // Edge lengths come from coordinate differences, which can amplify roundoff.
429 for (std::size_t component = 1; component < Source::mem_size(); ++component)
430 assert((Kokkos::abs(
431 stencil.weights
432 [component * Source::mem_size() + component]
433 - stencil.weights[0])
434 <= Kokkos::sqrt(std::numeric_limits<double>::epsilon()) * scale)
435 && "Scaling staged operator requires equal diagonal "
436 "coefficients");
437 }
438 }
439#endif
440 for (std::size_t entry = 0; entry < Index::mem_size(); ++entry) {
441 auto const ids = Index::mem_id_to_canonical_natural_ids(entry);
442 weights(node, entry) = stencil.weights[ids[0] * Source::mem_size() + ids[1]];
443 }
444 });
445 return storage;
446}
447
448struct NoStencilRule
449{
450};
451
452template <class Operator, class Geometry, class Input, class Element, class Rule1, class Rule2>
453KOKKOS_FORCEINLINE_FUNCTION auto stencil_value(
454 Geometry const& geometry,
455 Input const& input,
456 Element element,
457 Rule1 const& rule1,
458 Rule2 const& rule2)
459{
460 if constexpr (std::is_same_v<Rule2, NoStencilRule>)
461 return Operator::value(geometry, input, element, rule1);
462 else
463 return Operator::value(geometry, input, element, rule1, rule2);
464}
465
466template <
467 class Operator,
468 class Index,
469 class Source,
470 class Target,
471 std::size_t Radius,
472 class Exec,
473 class Tensor,
474 class Geometry,
475 class Rule1,
476 class Rule2 = NoStencilRule>
477StencilStorage<
478 Index,
479 Source,
480 Target,
481 Radius,
482 typename Tensor::non_indices_domain_t,
483 typename Tensor::memory_space>
484cache_stencils(
485 Exec const& exec,
486 Tensor const& input,
487 Geometry const& geometry,
488 Rule1 const& rule1,
489 Rule2 const& rule2 = {})
490{
491 using Domain = typename Tensor::non_indices_domain_t;
492 using Memory = typename Tensor::memory_space;
493 StencilStorage<Index, Source, Target, Radius, Domain, Memory> storage;
494 storage.domain = input.non_indices_domain();
495 storage.weights = decltype(storage.weights)(
496 Kokkos::view_alloc(Kokkos::WithoutInitializing, "staged_stencils"),
497 storage.domain.size());
498 storage.offset = decltype(storage.offset)(
499 Kokkos::view_alloc(Kokkos::WithoutInitializing, "staged_offsets"),
500 storage.domain.size());
501 storage.constant_response = decltype(storage.constant_response)(
502 Kokkos::view_alloc(Kokkos::WithoutInitializing, "staged_constant_response"),
503 storage.domain.size());
504 Kokkos::View<int, Memory> incomplete("staged_stencil_incomplete");
505 auto const weights = storage.weights;
506 auto const offset = storage.offset;
507 auto const constant_response = storage.constant_response;
508 Domain const domain = storage.domain;
509 Kokkos::parallel_for(
510 "similie_cache_operator_stencils",
511 Kokkos::RangePolicy<Exec>(exec, 0, domain.size()),
512 KOKKOS_LAMBDA(std::size_t node) {
513 auto const stencil = stencil_value<
514 Operator>(geometry, input, stencil_element(domain, node), rule1, rule2);
515 if (!stencil.complete)
516 Kokkos::atomic_max(&incomplete(), 1);
517 for (std::size_t target = 0; target < Target::mem_size(); ++target) {
518 offset(node, target) = stencil.offset[target];
519 for (std::size_t source = 0; source < Source::mem_size(); ++source)
520 constant_response(node, target, source)
521 = stencil.constant_response[target * Source::mem_size() + source];
522 }
523 for (std::size_t entry = 0; entry < stencil.weights.size(); ++entry)
524 weights(node, entry) = stencil.weights[entry];
525 });
526 int failed;
527 Kokkos::deep_copy(failed, incomplete);
528 exec.fence();
529 if (failed)
530 throw std::invalid_argument(
531 "Boundary closure exceeds the operator's fixed stencil support");
532 return storage;
533}
534
535template <
536 class Index,
537 class Source,
538 class Target,
539 std::size_t Radius,
540 class Exec,
541 class Output,
542 class Input,
543 class Domain,
544 class Memory>
545Output apply_stencils(
546 Exec const& exec,
547 Output const& output,
548 Input const& input,
549 StencilStorage<Index, Source, Target, Radius, Domain, Memory> const& storage)
550{
551 if (input.non_indices_domain() != storage.domain)
552 throw std::invalid_argument("Staged operator input domain changed");
553 auto const weights = storage.weights;
554 auto const offset = storage.offset;
555 auto const constant_response = storage.constant_response;
556 Domain const domain = storage.domain;
557 Domain const output_domain = output.non_indices_domain();
558 if (!output_domain.empty()
559 && (!misc::domain_contains(domain, output_domain.front())
560 || !misc::domain_contains(domain, output_domain.back())))
561 throw std::invalid_argument("Staged operator output domain exceeds its cached domain");
562 Kokkos::parallel_for(
563 "similie_apply_staged_stencil",
564 Kokkos::RangePolicy<Exec>(exec, 0, output_domain.size() * Target::mem_size()),
565 KOKKOS_LAMBDA(std::size_t row) {
566 auto const element = stencil_element(output_domain, row / Target::mem_size());
567 std::size_t const node = stencil_node(domain, element),
568 target = row % Target::mem_size();
569 std::array<double, Source::mem_size()> reference;
570 double sum = offset(node, target);
571 for (std::size_t source = 0; source < Source::mem_size(); ++source) {
572 reference[source] = input.mem(element, ddc::DiscreteElement<Source>(source));
573 sum += constant_response(node, target, source) * reference[source];
574 }
575 for (std::size_t entry
576 = target * StencilShape<Index, Radius>::size * Source::mem_size();
577 entry < (target + 1) * StencilShape<Index, Radius>::size * Source::mem_size();
578 ++entry) {
579 std::size_t const source = entry % Source::mem_size();
580 std::size_t const point
581 = (entry / Source::mem_size()) % StencilShape<Index, Radius>::size;
582 double const weight = weights(node, entry);
583 if (weight != 0.0)
584 sum += weight
585 * (input
586 .mem(stencil_source<
587 Index,
588 Radius>(domain, element, point),
589 ddc::DiscreteElement<Source>(source))
590 - reference[source]);
591 }
592 output.mem(element, ddc::DiscreteElement<Target>(target)) = sum;
593 });
594 return output;
595}
596
597} // namespace detail
598} // namespace sil::exterior
std::array< double, Target::mem_size()> offset
Definition stencil.hpp:32
std::array< double, Target::mem_size() *Source::mem_size()> constant_response
Definition stencil.hpp:33
Form(Coefficients const &) -> Form< Coefficients >
KOKKOS_FUNCTION constexpr auto tangent_basis(Elem elem)
std::array< double, Target::mem_size() *Source::mem_size() *detail::StencilShape< Index, Radius >::size > weights
Definition stencil.hpp:31
KOKKOS_FUNCTION bool domain_contains(ddc::DiscreteDomain< DDim... > const &dom, ddc::DiscreteElement< ODDim... > elem)
KOKKOS_FUNCTION constexpr std::size_t factorial(std::size_t k) noexcept
Definition factorial.hpp:13
Tensor(ddc::Chunk< ElementType, SupportType, Allocator >) -> Tensor< ElementType, SupportType, Kokkos::layout_right, typename Allocator::memory_space >
OwningTensor(TensorType, Storage &&) -> OwningTensor< TensorType, Storage >
Homogeneous bilinear coefficients, ordered output, first point/component, second point/component (sec...
Definition stencil.hpp:55
static constexpr std::size_t first_size
Definition stencil.hpp:57
std::array< double, Target::mem_size() *first_size *second_size > weights
Definition stencil.hpp:60
static constexpr std::size_t second_size
Definition stencil.hpp:59