Eigen-unsupported  3.4.90 (git rev a4098ac676528a83cfb73d4d26ce1b42ec05f47c)
TensorReverse.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Navdeep Jaitly <ndjaitly@google.com>
5// Benoit Steiner <benoit.steiner.goog@gmail.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_CXX11_TENSOR_TENSOR_REVERSE_H
12#define EIGEN_CXX11_TENSOR_TENSOR_REVERSE_H
13#include "./InternalHeaderCheck.h"
14
15namespace Eigen {
16
23namespace internal {
24template<typename ReverseDimensions, typename XprType>
25struct traits<TensorReverseOp<ReverseDimensions,
26 XprType> > : public traits<XprType>
27{
28 typedef typename XprType::Scalar Scalar;
29 typedef traits<XprType> XprTraits;
30 typedef typename XprTraits::StorageKind StorageKind;
31 typedef typename XprTraits::Index Index;
32 typedef typename XprType::Nested Nested;
33 typedef typename remove_reference<Nested>::type _Nested;
34 static const int NumDimensions = XprTraits::NumDimensions;
35 static const int Layout = XprTraits::Layout;
36 typedef typename XprTraits::PointerType PointerType;
37};
38
39template<typename ReverseDimensions, typename XprType>
40struct eval<TensorReverseOp<ReverseDimensions, XprType>, Eigen::Dense>
41{
42 typedef const TensorReverseOp<ReverseDimensions, XprType>& type;
43};
44
45template<typename ReverseDimensions, typename XprType>
46struct nested<TensorReverseOp<ReverseDimensions, XprType>, 1,
47 typename eval<TensorReverseOp<ReverseDimensions, XprType> >::type>
48{
49 typedef TensorReverseOp<ReverseDimensions, XprType> type;
50};
51
52} // end namespace internal
53
54template<typename ReverseDimensions, typename XprType>
55class TensorReverseOp : public TensorBase<TensorReverseOp<ReverseDimensions,
56 XprType>, WriteAccessors>
57{
58 public:
59 typedef TensorBase<TensorReverseOp<ReverseDimensions, XprType>, WriteAccessors>Base;
60 typedef typename Eigen::internal::traits<TensorReverseOp>::Scalar Scalar;
61 typedef typename Eigen::NumTraits<Scalar>::Real RealScalar;
62 typedef typename XprType::CoeffReturnType CoeffReturnType;
63 typedef typename Eigen::internal::nested<TensorReverseOp>::type Nested;
64 typedef typename Eigen::internal::traits<TensorReverseOp>::StorageKind
65 StorageKind;
66 typedef typename Eigen::internal::traits<TensorReverseOp>::Index Index;
67
68 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorReverseOp(
69 const XprType& expr, const ReverseDimensions& reverse_dims)
70 : m_xpr(expr), m_reverse_dims(reverse_dims) { }
71
72 EIGEN_DEVICE_FUNC
73 const ReverseDimensions& reverse() const { return m_reverse_dims; }
74
75 EIGEN_DEVICE_FUNC
76 const typename internal::remove_all<typename XprType::Nested>::type&
77 expression() const { return m_xpr; }
78
79 EIGEN_TENSOR_INHERIT_ASSIGNMENT_OPERATORS(TensorReverseOp)
80
81
82 protected:
83 typename XprType::Nested m_xpr;
84 const ReverseDimensions m_reverse_dims;
85};
86
87// Eval as rvalue
88template<typename ReverseDimensions, typename ArgType, typename Device>
89struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device>
90{
91 typedef TensorReverseOp<ReverseDimensions, ArgType> XprType;
92 typedef typename XprType::Index Index;
93 static const int NumDims = internal::array_size<ReverseDimensions>::value;
94 typedef DSizes<Index, NumDims> Dimensions;
95 typedef typename XprType::Scalar Scalar;
96 typedef typename XprType::CoeffReturnType CoeffReturnType;
97 typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
98 static const int PacketSize = PacketType<CoeffReturnType, Device>::size;
99 typedef StorageMemory<CoeffReturnType, Device> Storage;
100 typedef typename Storage::Type EvaluatorPointerType;
101
102 enum {
103 IsAligned = false,
104 PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
105 BlockAccess = NumDims > 0,
106 PreferBlockAccess = true,
107 Layout = TensorEvaluator<ArgType, Device>::Layout,
108 CoordAccess = false, // to be implemented
109 RawAccess = false
110 };
111
112 typedef internal::TensorIntDivisor<Index> IndexDivisor;
113
114 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
115 typedef internal::TensorBlockDescriptor<NumDims, Index> TensorBlockDesc;
116 typedef internal::TensorBlockScratchAllocator<Device> TensorBlockScratch;
117
118 typedef typename TensorEvaluator<const ArgType, Device>::TensorBlock
119 ArgTensorBlock;
120
121 typedef typename internal::TensorMaterializedBlock<CoeffReturnType, NumDims,
122 Layout, Index>
123 TensorBlock;
124 //===--------------------------------------------------------------------===//
125
126 EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
127 : m_impl(op.expression(), device),
128 m_reverse(op.reverse()),
129 m_device(device)
130 {
131 // Reversing a scalar isn't supported yet. It would be a no-op anyway.
132 EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE);
133
134 // Compute strides
135 m_dimensions = m_impl.dimensions();
136 if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
137 m_strides[0] = 1;
138 for (int i = 1; i < NumDims; ++i) {
139 m_strides[i] = m_strides[i-1] * m_dimensions[i-1];
140 if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]);
141 }
142 } else {
143 m_strides[NumDims-1] = 1;
144 for (int i = NumDims - 2; i >= 0; --i) {
145 m_strides[i] = m_strides[i+1] * m_dimensions[i+1];
146 if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]);
147 }
148 }
149 }
150
151 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
152 const Dimensions& dimensions() const { return m_dimensions; }
153
154 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) {
155 m_impl.evalSubExprsIfNeeded(NULL);
156 return true;
157 }
158
159#ifdef EIGEN_USE_THREADS
160 template <typename EvalSubExprsCallback>
161 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(
162 EvaluatorPointerType, EvalSubExprsCallback done) {
163 m_impl.evalSubExprsIfNeededAsync(nullptr, [done](bool) { done(true); });
164 }
165#endif // EIGEN_USE_THREADS
166
167 EIGEN_STRONG_INLINE void cleanup() {
168 m_impl.cleanup();
169 }
170
171 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index reverseIndex(
172 Index index) const {
173 eigen_assert(index < dimensions().TotalSize());
174 Index inputIndex = 0;
175 if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
176 EIGEN_UNROLL_LOOP
177 for (int i = NumDims - 1; i > 0; --i) {
178 Index idx = index / m_fastStrides[i];
179 index -= idx * m_strides[i];
180 if (m_reverse[i]) {
181 idx = m_dimensions[i] - idx - 1;
182 }
183 inputIndex += idx * m_strides[i] ;
184 }
185 if (m_reverse[0]) {
186 inputIndex += (m_dimensions[0] - index - 1);
187 } else {
188 inputIndex += index;
189 }
190 } else {
191 EIGEN_UNROLL_LOOP
192 for (int i = 0; i < NumDims - 1; ++i) {
193 Index idx = index / m_fastStrides[i];
194 index -= idx * m_strides[i];
195 if (m_reverse[i]) {
196 idx = m_dimensions[i] - idx - 1;
197 }
198 inputIndex += idx * m_strides[i] ;
199 }
200 if (m_reverse[NumDims-1]) {
201 inputIndex += (m_dimensions[NumDims-1] - index - 1);
202 } else {
203 inputIndex += index;
204 }
205 }
206 return inputIndex;
207 }
208
209 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(
210 Index index) const {
211 return m_impl.coeff(reverseIndex(index));
212 }
213
214 template<int LoadMode>
215 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
216 PacketReturnType packet(Index index) const
217 {
218 EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE)
219 eigen_assert(index+PacketSize-1 < dimensions().TotalSize());
220
221 // TODO(ndjaitly): write a better packing routine that uses
222 // local structure.
223 EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type
224 values[PacketSize];
225 EIGEN_UNROLL_LOOP
226 for (int i = 0; i < PacketSize; ++i) {
227 values[i] = coeff(index+i);
228 }
229 PacketReturnType rslt = internal::pload<PacketReturnType>(values);
230 return rslt;
231 }
232
233 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
234 internal::TensorBlockResourceRequirements getResourceRequirements() const {
235 const size_t target_size = m_device.lastLevelCacheSize();
236 // Block evaluation reads underlying memory in reverse order, and default
237 // cost model does not properly catch this in bytes stored/loaded.
238 return internal::TensorBlockResourceRequirements::skewed<Scalar>(
239 target_size)
240 .addCostPerCoeff({0, 0, 24});
241 }
242
243 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock
244 block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
245 bool /*root_of_expr_ast*/ = false) const {
246 // TODO(ezhulenev): If underlying tensor expression supports and prefers
247 // block evaluation we must use it. Currently we use coeff and packet
248 // access into the underlying tensor expression.
249 // static const bool useBlockAccessForArgType =
250 // TensorEvaluator<ArgType, Device>::BlockAccess &&
251 // TensorEvaluator<ArgType, Device>::PreferBlockAccess;
252
253 static const bool isColMajor =
254 static_cast<int>(Layout) == static_cast<int>(ColMajor);
255
256 static const Index inner_dim_idx = isColMajor ? 0 : NumDims - 1;
257 const bool inner_dim_reversed = m_reverse[inner_dim_idx];
258
259 // Offset in the output block.
260 Index block_offset = 0;
261
262 // Offset in the input Tensor.
263 Index input_offset = reverseIndex(desc.offset());
264
265 // Initialize output block iterator state. Dimension in this array are
266 // always in inner_most -> outer_most order (col major layout).
267 array<BlockIteratorState, NumDims> it;
268 for (int i = 0; i < NumDims; ++i) {
269 const int dim = isColMajor ? i : NumDims - 1 - i;
270 it[i].size = desc.dimension(dim);
271 it[i].count = 0;
272 it[i].reverse = m_reverse[dim];
273
274 it[i].block_stride =
275 i == 0 ? 1 : (it[i - 1].size * it[i - 1].block_stride);
276 it[i].block_span = it[i].block_stride * (it[i].size - 1);
277
278 it[i].input_stride = m_strides[dim];
279 it[i].input_span = it[i].input_stride * (it[i].size - 1);
280
281 if (it[i].reverse) {
282 it[i].input_stride = -1 * it[i].input_stride;
283 it[i].input_span = -1 * it[i].input_span;
284 }
285 }
286
287 // If multiple inner dimensions have the same reverse flag, check if we can
288 // merge them into a single virtual inner dimension.
289 int effective_inner_dim = 0;
290 for (int i = 1; i < NumDims; ++i) {
291 if (it[i].reverse != it[effective_inner_dim].reverse) break;
292 if (it[i].block_stride != it[effective_inner_dim].size) break;
293 if (it[i].block_stride != numext::abs(it[i].input_stride)) break;
294
295 it[i].size = it[effective_inner_dim].size * it[i].size;
296
297 it[i].block_stride = 1;
298 it[i].input_stride = (inner_dim_reversed ? -1 : 1);
299
300 it[i].block_span = it[i].block_stride * (it[i].size - 1);
301 it[i].input_span = it[i].input_stride * (it[i].size - 1);
302
303 effective_inner_dim = i;
304 }
305
306 eigen_assert(it[effective_inner_dim].block_stride == 1);
307 eigen_assert(it[effective_inner_dim].input_stride ==
308 (inner_dim_reversed ? -1 : 1));
309
310 const Index inner_dim_size = it[effective_inner_dim].size;
311
312 // Prepare storage for the materialized reverse result.
313 const typename TensorBlock::Storage block_storage =
314 TensorBlock::prepareStorage(desc, scratch);
315 CoeffReturnType* block_buffer = block_storage.data();
316
317 while (it[NumDims - 1].count < it[NumDims - 1].size) {
318 // Copy inner-most dimension data from reversed location in input.
319 Index dst = block_offset;
320 Index src = input_offset;
321
322 // NOTE(ezhulenev): Adding vectorized path with internal::preverse showed
323 // worse results in benchmarks than a simple coefficient loop.
324 if (inner_dim_reversed) {
325 for (Index i = 0; i < inner_dim_size; ++i) {
326 block_buffer[dst] = m_impl.coeff(src);
327 ++dst;
328 --src;
329 }
330 } else {
331 for (Index i = 0; i < inner_dim_size; ++i) {
332 block_buffer[dst] = m_impl.coeff(src);
333 ++dst;
334 ++src;
335 }
336 }
337
338 // For the 1d tensor we need to generate only one inner-most dimension.
339 if ((NumDims - effective_inner_dim) == 1) break;
340
341 // Update offset.
342 for (Index i = effective_inner_dim + 1; i < NumDims; ++i) {
343 if (++it[i].count < it[i].size) {
344 block_offset += it[i].block_stride;
345 input_offset += it[i].input_stride;
346 break;
347 }
348 if (i != NumDims - 1) it[i].count = 0;
349 block_offset -= it[i].block_span;
350 input_offset -= it[i].input_span;
351 }
352 }
353
354 return block_storage.AsTensorMaterializedBlock();
355 }
356
357 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
358 double compute_cost = NumDims * (2 * TensorOpCost::AddCost<Index>() +
359 2 * TensorOpCost::MulCost<Index>() +
360 TensorOpCost::DivCost<Index>());
361 for (int i = 0; i < NumDims; ++i) {
362 if (m_reverse[i]) {
363 compute_cost += 2 * TensorOpCost::AddCost<Index>();
364 }
365 }
366 return m_impl.costPerCoeff(vectorized) +
367 TensorOpCost(0, 0, compute_cost, false /* vectorized */, PacketSize);
368 }
369
370 EIGEN_DEVICE_FUNC typename Storage::Type data() const { return NULL; }
371
372#ifdef EIGEN_USE_SYCL
373 // binding placeholder accessors to a command group handler for SYCL
374 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void bind(cl::sycl::handler &cgh) const {
375 m_impl.bind(cgh);
376 }
377#endif
378
379 protected:
380 Dimensions m_dimensions;
381 array<Index, NumDims> m_strides;
382 array<IndexDivisor, NumDims> m_fastStrides;
383 TensorEvaluator<ArgType, Device> m_impl;
384 ReverseDimensions m_reverse;
385 const Device EIGEN_DEVICE_REF m_device;
386
387 private:
388 struct BlockIteratorState {
389 BlockIteratorState()
390 : size(0),
391 count(0),
392 reverse(false),
393 block_stride(0),
394 block_span(0),
395 input_stride(0),
396 input_span(0) {}
397
398 Index size;
399 Index count;
400 bool reverse;
401 Index block_stride;
402 Index block_span;
403 Index input_stride;
404 Index input_span;
405 };
406};
407
408// Eval as lvalue
409
410template <typename ReverseDimensions, typename ArgType, typename Device>
411struct TensorEvaluator<TensorReverseOp<ReverseDimensions, ArgType>, Device>
412 : public TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>,
413 Device> {
414 typedef TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>,
415 Device> Base;
416 typedef TensorReverseOp<ReverseDimensions, ArgType> XprType;
417 typedef typename XprType::Index Index;
418 static const int NumDims = internal::array_size<ReverseDimensions>::value;
419 typedef DSizes<Index, NumDims> Dimensions;
420
421 enum {
422 IsAligned = false,
423 PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
424 BlockAccess = false,
425 PreferBlockAccess = false,
426 Layout = TensorEvaluator<ArgType, Device>::Layout,
427 CoordAccess = false, // to be implemented
428 RawAccess = false
429 };
430 EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
431 : Base(op, device) {}
432
433 typedef typename XprType::Scalar Scalar;
434 typedef typename XprType::CoeffReturnType CoeffReturnType;
435 typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
436 static const int PacketSize = PacketType<CoeffReturnType, Device>::size;
437
438 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
439 typedef internal::TensorBlockNotImplemented TensorBlock;
440 //===--------------------------------------------------------------------===//
441
442 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
443 const Dimensions& dimensions() const { return this->m_dimensions; }
444
445 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) {
446 return this->m_impl.coeffRef(this->reverseIndex(index));
447 }
448
449 template <int StoreMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
450 void writePacket(Index index, const PacketReturnType& x) {
451 EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE)
452 eigen_assert(index+PacketSize-1 < dimensions().TotalSize());
453
454 // This code is pilfered from TensorMorphing.h
455 EIGEN_ALIGN_MAX CoeffReturnType values[PacketSize];
456 internal::pstore<CoeffReturnType, PacketReturnType>(values, x);
457 EIGEN_UNROLL_LOOP
458 for (int i = 0; i < PacketSize; ++i) {
459 this->coeffRef(index+i) = values[i];
460 }
461 }
462};
463
464
465} // end namespace Eigen
466
467#endif // EIGEN_CXX11_TENSOR_TENSOR_REVERSE_H
WriteAccessors
Namespace containing all symbols from the Eigen library.
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index