Cutlass
CUDA Templates for Linear Algebra Subroutines and Solvers
fragment_stream.h
Go to the documentation of this file.
1 /***************************************************************************************************
2  * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without modification, are permitted
5  * provided that the following conditions are met:
6  * * Redistributions of source code must retain the above copyright notice, this list of
7  * conditions and the following disclaimer.
8  * * Redistributions in binary form must reproduce the above copyright notice, this list of
9  * conditions and the following disclaimer in the documentation and/or other materials
10  * provided with the distribution.
11  * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
12  * to endorse or promote products derived from this software without specific prior written
13  * permission.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
17  * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
19  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
21  * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  *
24  **************************************************************************************************/
29 #pragma once
30 
31 #include <cutlass/convert.h>
32 #include <cutlass/fragment.h>
34 #include <cutlass/tensor_ref.h>
35 #include <cutlass/tile_iterator.h>
36 
39 #include <cutlass/matrix_traits.h>
40 
41 namespace cutlass {
42 
44 
46 template <typename Traits_, typename LoadIterator_, typename StoreIterator_,
47  typename Convert_ =
48  FragmentCopy<typename StoreIterator_::Fragment, typename LoadIterator_::Fragment>,
49  typename Index_ = int>
51  //
52  // Constant and type definitions
53  //
54 
56  typedef Traits_ Traits;
57 
59  typedef LoadIterator_ LoadIterator;
60 
62  typedef StoreIterator_ StoreIterator;
63 
65  typedef Convert_ Convert;
66 
68  typedef Index_ Index;
69 
70  //
71  // Dependent types
72  //
73 
75  typedef typename LoadIterator::Fragment Fragment;
76 
79 
81  typedef typename StoreIterator::Storage Storage;
82 
85 
86  //
87  // Nested classes
88  //
89 
91  struct Params {
93  typedef typename LoadIterator::Params LoadParams;
94 
96  typedef typename StoreIterator::Params StoreParams;
97 
98  //
99  // Data members
100  //
101 
104 
107 
108  //
109  // Methods
110  //
111 
113  CUTLASS_HOST_DEVICE int initialize(LoadParams const &_load_params,
114  StoreParams const &_store_params) {
115  load_params = _load_params;
116  store_params = _store_params;
117  return 0;
118  }
119  };
120 
121  //
122  // Data members
123  //
124 
127 
129  typename LoadIterator::PredicateVector predicates;
130 
133 
136 
139 
140  //
141  // Static members
142  //
143 
145  static CUTLASS_DEVICE void shared_store_fence() { __syncthreads(); }
146 
147  //
148  // Methods
149  //
150 
151  CUTLASS_DEVICE
153 
155  CUTLASS_DEVICE
156  FragmentStream(Params const &params, Coord<3> const &bounds,
157  Coord<3> const &block_offset = make_Coord(0, 0, 0))
158  : load_iterator(params.load_params, block_offset), store_iterator(params.store_params) {
159  // set predicates
160  initialize_predicates(bounds, block_offset);
161 
162  fetch.clear();
163  }
164 
166  CUTLASS_DEVICE
167  void load() {
168  ConstPredicateTileAdapter<typename LoadIterator::PredicateVector,
169  typename LoadIterator::Iterations>
170  predicates_it(predicates);
172  }
173 
175  CUTLASS_DEVICE
176  void commit() {
177  StoreFragment store_fragment(convert(fetch));
178 
179  iterator_store(store_iterator, store_fragment);
180  }
181 
183  CUTLASS_DEVICE
184  void initialize_predicates(Coord<3> const &bounds, Coord<3> const &block_offset) {
186  predicates_it(predicates);
187  load_iterator.initialize_predicates(predicates_it, bounds, block_offset);
188  }
189 };
190 
192 
193 } // namespace cutlass
StoreIterator store_iterator
Stores fragment to shared memory.
Definition: fragment_stream.h:132
Definition: convert.h:34
Defines a structure containing strides, bounds, and a pointer to tensor data.
StoreParams store_params
Parameters to the store iterator.
Definition: fragment_stream.h:106
LoadIterator::PredicateVector predicates
Predicate vector.
Definition: fragment_stream.h:129
Defines the Tile Traits concept and iterators for loading and storing to tiles efficiently.
Defines structural properties of complete GEMM computation.
CUTLASS_HOST_DEVICE Coord< 1 > make_Coord(int _0)
Helper to make a 2-element coordinate.
Definition: coord.h:241
StoreIterator::Storage Storage
Destination storage.
Definition: fragment_stream.h:81
CUTLASS_DEVICE FragmentStream(Params const &params, Coord< 3 > const &bounds, Coord< 3 > const &block_offset=make_Coord(0, 0, 0))
Constructor.
Definition: fragment_stream.h:156
Fragment fetch
Fragment fetched by load iterator.
Definition: fragment_stream.h:135
CUTLASS_DEVICE void commit()
Commits the fragment.
Definition: fragment_stream.h:176
StoreIterator_ StoreIterator
Defines the store iterator.
Definition: fragment_stream.h:62
Adapter to enable random access to predicates via logical coordinate within a tile.
Definition: predicate_vector.h:435
Index_ Index
Index type.
Definition: fragment_stream.h:68
Fragment< Scalar, ShapeCount< Tile >::kCount, kFragmentSize > Storage
The storage.
Definition: tile_iterator.h:181
static CUTLASS_DEVICE void shared_store_fence()
The memory fence for shared stores.
Definition: fragment_stream.h:145
Free functions for loading and storing to implementations of tile iteartor concepts.
CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, Fragment &fragment, typename InputIterator::Index offset, ConstPredicateAdapter predicate_adapter)
Loads a fragment from an input iterator, masked by a predicate iterator.
Definition: iterator_access.h:113
StoreIterator::Storage SharedStoreStorage
The storage in shared memory.
Definition: fragment_stream.h:84
CUTLASS_DEVICE void initialize_predicates(Coord< 3 > const &bounds, Coord< 3 > const &block_offset)
Recomputes predicates.
Definition: fragment_stream.h:184
Defines constant expressions for mapping GEMM problem size and strides onto pitch-linear memory...
Manages a pair of iterators to stream data from global memory to shared.
Definition: fragment_stream.h:50
CUTLASS_DEVICE FragmentStream()
Definition: fragment_stream.h:152
#define CUTLASS_HOST_DEVICE
Definition: cutlass.h:37
Statically-sized array specifying Coords within a tensor.
Definition: coord.h:48
CUTLASS_HOST_DEVICE void iterator_store(OutputIterator &iterator, Fragment &fragment)
Stores a fragment to an output iterator.
Definition: iterator_access.h:174
LoadIterator::Params LoadParams
Load parameters.
Definition: fragment_stream.h:93
Convert_ Convert
Converts between tiles.
Definition: fragment_stream.h:65
StoreIterator::Fragment StoreFragment
Stored fragment type.
Definition: fragment_stream.h:78
Traits_ Traits
Defines traits of WMMA GEMM tile stream.
Definition: fragment_stream.h:56
CUTLASS_DEVICE void load()
Loads the fragment.
Definition: fragment_stream.h:167
StoreIterator::Params StoreParams
Store parameters.
Definition: fragment_stream.h:96
Adapter to enable random access to predicates via logical coordinate within a tile.
Definition: predicate_vector.h:466
LoadIterator::Fragment Fragment
Loaded fragment type.
Definition: fragment_stream.h:75
LoadIterator_ LoadIterator
Defines the load iterator.
Definition: fragment_stream.h:59
CUTLASS_HOST_DEVICE int initialize(LoadParams const &_load_params, StoreParams const &_store_params)
Initializes parameters.
Definition: fragment_stream.h:113
Defines properties of matrices used to denote layout and operands to GEMM kernels.
LoadParams load_params
Parameters to load iterator.
Definition: fragment_stream.h:103
Convert convert
Converts between load fragments and store fragments.
Definition: fragment_stream.h:138
Defines Fragment, a statically-sized array for storing parts of matrices within a thread&#39;s registers...
LoadIterator load_iterator
Loads fragment from global memory.
Definition: fragment_stream.h:126
Defines conversion operations among Fragments of different base type.
Parameters passed to initialize the ierator.
Definition: fragment_stream.h:91