Namespaces
namespace	cuda
namespace	util
namespace	rocm

Classes
class	BarrierLocalNodeDispatcher
class	GpuArch
struct	FoldBarrier
struct	NestedFoldPlan
class	GpuMmaDispatcherStrategy
	Hardware-dependent emit primitives for the composable MMA library nodes. More...
class	GpuMmaNodeDispatcher
	Shared storage for the strategy; each composable MMA node has its own dispatcher subclass. More...
class	GpuMmaMatmulDispatcher
	A single MMA accumulation of A * B into a preinitialized accumulator (gpu::GpuMmaMatmul). More...
class	GpuMmaFillDispatcher
	Declare and zero-initialize an MMA accumulator fragment (gpu::GpuMmaFill). More...
class	GpuMmaFragmentLoadDispatcher
	Load a tile from memory into a fragment (gpu::GpuMmaFragmentLoad). More...
class	GpuMmaFragmentStoreDispatcher
	Store a fragment to a tile in memory (gpu::GpuMmaFragmentStore). More...
class	GpuMmaEltwiseAddDispatcher
	D = out_type(Acc + acc_type(C)), element-wise over the fragments (gpu::GpuMmaFragmentEltwiseAdd). More...
class	GpuMmaEinsumReplacer
class	GpuMmaEinsumTransform
class	GpuMmaExpander
class	GpuMmaFillNode
	Prepares (declares and zero-initializes) a GPU MMA accumulator fragment. More...
class	GpuMmaFillNodeSerializer
struct	MmaBlockSize
struct	BlockCounts
	Grouping of MMA blocks. More...
struct	GpuMmaTiling
struct	GpuMmaSupport
struct	GpuMmaFromMemoryLayout
	Describes how a single MMA fragment maps to memory relative to a base pointer. More...
class	GpuMmaFragmentEltwiseAddNode
	Handles the addition of two MMA fragments and stores the result in a third fragment. More...
class	GpuMmaFragmentEltwiseAddNodeSerializer
class	GpuMmaFragmentLoadNode
	Loads a single MMA fragment from memory. More...
class	GpuMmaFragmentLoadNodeSerializer
class	GpuMmaFragmentStoreNode
	Stores a single MMA fragment to memory. More...
class	GpuMmaFragmentStoreNodeSerializer
class	GpuMmaMatmulNode
	A single GPU tensor-core matrix-multiply-accumulate (MMA) step. More...
class	GpuMmaMatmulNodeSerializer
class	GPUOffloadDispatcherStrategy
class	GPUOffloadBaseDispatcher
	Shared scaffolding for the GPU offload dispatchers (Map and Reduce). More...
class	GPUOffloadMapDispatcher
	Map node policy for the GPU offload dispatcher. More...
class	GPUOffloadReduceDispatcher
class	ScheduleType_GPU_Offload
	Base class for GPU schedule types (CUDA/ROCm) using CRTP pattern. More...
class	GPUReduceDispatcher
	Atomics-based GPU code generator for structured_control_flow::Reduce. More...
struct	ReductionLayout
	Dense mixed-radix indexing for a strided reduction output footprint. More...
class	ScheduleType_GPU_Base
	Base class for GPU schedule types (CUDA/ROCm) using CRTP pattern. More...
class	GpuTileArrivalNode
	Counts the arrival of a thread block at a shared work item and elects the last arriving block. More...
class	GpuTileArrivalNodeSerializer
class	GpuTileArrivalDispatcher
	Emits GpuTileArrivalNode for the CUDA and ROCm kernel language extensions. More...
class	GPUOffloadTransform

Enumerations
enum class	MmaFragmentType { A = 1 , B = 2 , C = 3 }
enum	MmaFragmentLayout { MMA_LAYOUT_UNSPECIFIED = 0 , MMA_LAYOUT_ROW_MAJOR = 1 , MMA_LAYOUT_COL_MAJOR = 2 }
enum class	TargetLevel { X_GRID , Y_GRID , Z_GRID , X_BLOCK , Y_BLOCK , Z_BLOCK , WARP }
enum class	ReduceStrategy { Register , Shared , Global }
	Where a reduction's per-thread partials live, and hence how they combine. More...
enum	GPUDimension { X = 0 , Y = 1 , Z = 2 }
	Shared GPU dimension enum for CUDA/ROCm targets Represents the X, Y, Z dimensions used for GPU kernel launches. More...

Functions
template<typename ScheduleT>
symbolic::Expression	find_nested_gpu_blocksize (structured_control_flow::Map &node, analysis::AnalysisManager &analysis_manager, GPUDimension dimension)
	GPU map utility functions shared between CUDA and ROCm.
template<typename ScheduleT>
symbolic::Expression	find_nested_gpu_iterations (structured_control_flow::Map &node, analysis::AnalysisManager &analysis_manager, GPUDimension dimension)
	Find the number of iterations for nested GPU maps in a given dimension.
template<typename ScheduleT>
bool	is_outermost_gpu_map (structured_control_flow::Map &node, analysis::AnalysisManager &analysis_manager)
	Check if a map is the outermost GPU map in the loop tree.
template<typename ScheduleT>
symbolic::SymbolSet	get_gpu_indvars (structured_control_flow::Map &node, analysis::AnalysisManager &analysis_manager, GPUDimension dimension)
	Get all induction variables for GPU maps in a given dimension.
bool	is_gpu_schedule (const structured_control_flow::ScheduleType &schedule)
	Check if a schedule type is a GPU schedule (CUDA or ROCM).
template<typename GPUType>
int64_t	gpu_warp_size ()
template<>
int64_t	gpu_warp_size< sdfg::cuda::ScheduleType_CUDA_Offload > ()
template<>
int64_t	gpu_warp_size< sdfg::rocm::ScheduleType_ROCM_Offload > ()
int64_t	gpu_warp_size (const structured_control_flow::ScheduleType &schedule)
template<typename ScheduleT>
std::vector< structured_control_flow::Map * >	get_gpu_maps (structured_control_flow::Map &node, analysis::AnalysisManager &analysis_manager, GPUDimension dimension)
	Get all GPU Map nodes in a given dimension (in tree traversal order).
bool	nested_parallelization_is_unsafe (structured_control_flow::StructuredLoop &loop, analysis::AnalysisManager &analysis_manager)
	Whether parallelizing `loop` to a new GPU grid dimension would race on a shared container.
NestedFoldPlan	analyze_nested_fold (structured_control_flow::StructuredLoop &loop, TargetLevel target_level, analysis::AnalysisManager &analysis_manager)
symbolic::Expression	get_target_level_dim (TargetLevel target_level, int warp_size)
symbolic::Expression	get_target_level_idx (TargetLevel target_level)
bool	is_grid_level (TargetLevel target_level)
bool	is_block_level (TargetLevel target_level)
bool	is_warp_level (TargetLevel target_level)
symbolic::SymbolSet	target_level_indvars (structured_control_flow::StructuredLoop &node, analysis::AnalysisManager &analysis_manager, TargetLevel target_level)
void	get_nested_schedule_types (structured_control_flow::StructuredLoop &node, analysis::AnalysisManager &analysis_manager, std::unordered_map< TargetLevel, structured_control_flow::ScheduleType > &output)
void	get_nested_level_maps (structured_control_flow::StructuredLoop &node, analysis::AnalysisManager &analysis_manager, std::unordered_map< TargetLevel, structured_control_flow::StructuredLoop * > &output)
bool	nested_warp_dim (structured_control_flow::StructuredLoop &loop, analysis::AnalysisManager &analysis_manager)
void	validate_wave_maps (structured_control_flow::StructuredLoop &node, analysis::AnalysisManager &analysis_manager, int64_t warp_size)
structured_control_flow::StructuredLoop *	find_x_block_owning_warp_level (structured_control_flow::StructuredLoop &node, analysis::AnalysisManager &analysis_manager)
size_t	perfectly_nested_depth (structured_control_flow::StructuredLoop *loop)
template symbolic::Expression	find_nested_gpu_blocksize< sdfg::cuda::ScheduleType_CUDA > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
template symbolic::Expression	find_nested_gpu_blocksize< sdfg::rocm::ScheduleType_ROCM > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
template symbolic::Expression	find_nested_gpu_iterations< sdfg::cuda::ScheduleType_CUDA > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
template symbolic::Expression	find_nested_gpu_iterations< sdfg::rocm::ScheduleType_ROCM > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
template bool	is_outermost_gpu_map< sdfg::cuda::ScheduleType_CUDA > (structured_control_flow::Map &, analysis::AnalysisManager &)
template bool	is_outermost_gpu_map< sdfg::rocm::ScheduleType_ROCM > (structured_control_flow::Map &, analysis::AnalysisManager &)
template symbolic::SymbolSet	get_gpu_indvars< sdfg::cuda::ScheduleType_CUDA > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
template symbolic::SymbolSet	get_gpu_indvars< sdfg::rocm::ScheduleType_ROCM > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
template std::vector< structured_control_flow::Map * >	get_gpu_maps< sdfg::cuda::ScheduleType_CUDA > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
template std::vector< structured_control_flow::Map * >	get_gpu_maps< sdfg::rocm::ScheduleType_ROCM > (structured_control_flow::Map &, analysis::AnalysisManager &, GPUDimension)
void	emit_clipped_fragment_transfer (codegen::CodegenOutput &out, const GpuMmaDispatcherStrategy &strategy, bool load, const std::string &frag, const std::string &base, const GpuMmaFromMemoryLayout &layout, const MmaBlockSize &block_size, MmaFragmentType frag_type, types::PrimitiveType element_type, MmaFragmentLayout fragment_layout)
	Emit a fragment load/store whose layout is clipped (GpuMmaFromMemoryLayout::rows/cols): a wave-uniform branch takes the plain transfer when the fragment is fully in bounds, otherwise an element-wise transfer that zero-fills out-of-bounds loads and skips out-of-bounds stores.
data_flow::LibraryNodeCode	LibraryNodeType_GpuMmaFill ("gpu::GpuMmaFill")
std::ostream &	operator<< (std::ostream &os, const MmaBlockSize &block_size)
constexpr const char *	mma_fragment_layout_to_string (MmaFragmentLayout layout)
std::ostream &	operator<< (std::ostream &os, const GpuMmaFromMemoryLayout &layout)
void	serialize_mma_block_size (nlohmann::json &j, const MmaBlockSize &block_size)
MmaBlockSize	deserialize_mma_block_size (const nlohmann::json &j)
void	serialize_mma_from_memory_layout (nlohmann::json &j, const GpuMmaFromMemoryLayout &layout)
GpuMmaFromMemoryLayout	deserialize_mma_from_memory_layout (const nlohmann::json &j)
data_flow::LibraryNodeCode	LibraryNodeType_GpuMmaFragmentEltwiseAdd ("gpu::MmaFragmentEltwiseAdd")
data_flow::LibraryNodeCode	LibraryNodeType_GpuMmaFragmentLoad ("gpu::MmaFragmentLoad")
data_flow::LibraryNodeCode	LibraryNodeType_GpuMmaFragmentStore ("gpu::MmaFragmentStore")
data_flow::LibraryNodeCode	LibraryNodeType_GpuMmaMatmul ("gpu::GpuMmaMatmul")
std::string	to_string (const TargetLevel &target_level)
TargetLevel	target_level_from_string (const std::string &value)
std::string	reduce_strategy_to_string (ReduceStrategy strategy)
ReduceStrategy	reduce_strategy_from_string (const std::string &value)
ReduceStrategy	default_reduce_strategy (TargetLevel target_level)
TargetLevel	gpu_target_level (const structured_control_flow::ScheduleType &schedule)
	Get the GPU target level from any GPU schedule type.
void	gpu_target_level (structured_control_flow::ScheduleType &schedule, const TargetLevel &target_level)
	Set the GPU target level on any GPU schedule type.
GPUDimension	gpu_dimension (const structured_control_flow::ScheduleType &schedule)
	Get the GPU dimension from any GPU schedule type.
void	gpu_dimension (structured_control_flow::ScheduleType &schedule, const GPUDimension &dimension)
	Set the GPU dimension on any GPU schedule type.
symbolic::Integer	gpu_block_size (const structured_control_flow::ScheduleType &schedule)
	Get the block size from any GPU schedule type Returns default values if not explicitly set:
void	gpu_block_size (structured_control_flow::ScheduleType &schedule, const symbolic::Expression block_size)
	Set the block size on any GPU schedule type.
data_flow::LibraryNodeCode	LibraryNodeType_GpuTileArrival ("gpu::TileArrival")

Enumeration Type Documentation

◆ GPUDimension

enum sdfg::gpu::GPUDimension

Shared GPU dimension enum for CUDA/ROCm targets Represents the X, Y, Z dimensions used for GPU kernel launches.

Enumerator
X
Y
Z

◆ MmaFragmentLayout

enum sdfg::gpu::MmaFragmentLayout

Enumerator
MMA_LAYOUT_UNSPECIFIED
MMA_LAYOUT_ROW_MAJOR
MMA_LAYOUT_COL_MAJOR

◆ MmaFragmentType

enum class sdfg::gpu::MmaFragmentType

strong

Enumerator
A
B
C

◆ ReduceStrategy

enum class sdfg::gpu::ReduceStrategy

strong

Where a reduction's per-thread partials live, and hence how they combine.

Orthogonal to the TargetLevel (which fixes the cooperation scope): the strategy fixes the mechanism.

Register: per-thread/lane register; a warp scope folds via __shfl_xor_sync.
Shared: NV_Shared buffer + block halving tree.
Global: per-thread register merged atomically into the global accumulator.

Enumerator
Register
Shared
Global

◆ TargetLevel

enum class sdfg::gpu::TargetLevel

strong

Enumerator
X_GRID
Y_GRID
Z_GRID
X_BLOCK
Y_BLOCK
Z_BLOCK
WARP

Function Documentation

◆ analyze_nested_fold()

NestedFoldPlan sdfg::gpu::analyze_nested_fold	(	structured_control_flow::StructuredLoop &	loop,
		TargetLevel	target_level,
		analysis::AnalysisManager &	analysis_manager )

◆ default_reduce_strategy()

ReduceStrategy sdfg::gpu::default_reduce_strategy ( TargetLevel target_level )

inline

◆ deserialize_mma_block_size()

MmaBlockSize sdfg::gpu::deserialize_mma_block_size ( const nlohmann::json & j )

◆ deserialize_mma_from_memory_layout()

GpuMmaFromMemoryLayout sdfg::gpu::deserialize_mma_from_memory_layout ( const nlohmann::json & j )

◆ emit_clipped_fragment_transfer()

void sdfg::gpu::emit_clipped_fragment_transfer	(	codegen::CodegenOutput &	out,
		const GpuMmaDispatcherStrategy &	strategy,
		bool	load,
		const std::string &	frag,
		const std::string &	base,
		const GpuMmaFromMemoryLayout &	layout,
		const MmaBlockSize &	block_size,
		MmaFragmentType	frag_type,
		types::PrimitiveType	element_type,
		MmaFragmentLayout	fragment_layout )

Emit a fragment load/store whose layout is clipped (GpuMmaFromMemoryLayout::rows/cols): a wave-uniform branch takes the plain transfer when the fragment is fully in bounds, otherwise an element-wise transfer that zero-fills out-of-bounds loads and skips out-of-bounds stores.

The element coordinates come from loading an iota table through the strategy's own fragment load, so no knowledge of the fragment's register layout is needed.

◆ find_nested_gpu_blocksize()

template<typename ScheduleT>

symbolic::Expression sdfg::gpu::find_nested_gpu_blocksize	(	structured_control_flow::Map &	node,
		analysis::AnalysisManager &	analysis_manager,
		GPUDimension	dimension )

GPU map utility functions shared between CUDA and ROCm.

These template functions provide common functionality for GPU map dispatchers. They are parameterized by the ScheduleType class (ScheduleType_CUDA or ScheduleType_ROCM).

Template Parameters

ScheduleT The schedule type class (cuda::ScheduleType_CUDA or rocm::ScheduleType_ROCM)

Find the block size for nested GPU maps in a given dimension

Template Parameters

ScheduleT Schedule type class with value(), dimension(), and block_size() static methods

Parameters

node	The current map node
analysis_manager	Analysis manager for loop analysis
dimension	GPU dimension (X, Y, or Z)

Returns: Block size expression, or symbolic::one() if not found

◆ find_nested_gpu_blocksize< sdfg::cuda::ScheduleType_CUDA >()

template symbolic::Expression sdfg::gpu::find_nested_gpu_blocksize< sdfg::cuda::ScheduleType_CUDA >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ find_nested_gpu_blocksize< sdfg::rocm::ScheduleType_ROCM >()

template symbolic::Expression sdfg::gpu::find_nested_gpu_blocksize< sdfg::rocm::ScheduleType_ROCM >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ find_nested_gpu_iterations()

template<typename ScheduleT>

symbolic::Expression sdfg::gpu::find_nested_gpu_iterations	(	structured_control_flow::Map &	node,
		analysis::AnalysisManager &	analysis_manager,
		GPUDimension	dimension )

Find the number of iterations for nested GPU maps in a given dimension.

Template Parameters

ScheduleT Schedule type class with value() and dimension() static methods

Parameters

node	The current map node
analysis_manager	Analysis manager for loop and assumptions analysis
dimension	GPU dimension (X, Y, or Z)

Returns: Number of iterations expression, or symbolic::one() if not found

◆ find_nested_gpu_iterations< sdfg::cuda::ScheduleType_CUDA >()

template symbolic::Expression sdfg::gpu::find_nested_gpu_iterations< sdfg::cuda::ScheduleType_CUDA >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ find_nested_gpu_iterations< sdfg::rocm::ScheduleType_ROCM >()

template symbolic::Expression sdfg::gpu::find_nested_gpu_iterations< sdfg::rocm::ScheduleType_ROCM >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ find_x_block_owning_warp_level()

structured_control_flow::StructuredLoop * sdfg::gpu::find_x_block_owning_warp_level	(	structured_control_flow::StructuredLoop &	node,
		analysis::AnalysisManager &	analysis_manager )

◆ get_gpu_indvars()

template<typename ScheduleT>

symbolic::SymbolSet sdfg::gpu::get_gpu_indvars	(	structured_control_flow::Map &	node,
		analysis::AnalysisManager &	analysis_manager,
		GPUDimension	dimension )

Get all induction variables for GPU maps in a given dimension.

Template Parameters

ScheduleT Schedule type class with value() and dimension() static methods

Parameters

node	The current map node
analysis_manager	Analysis manager for loop analysis
dimension	GPU dimension (X, Y, or Z)

Returns: Set of induction variable symbols

◆ get_gpu_indvars< sdfg::cuda::ScheduleType_CUDA >()

template symbolic::SymbolSet sdfg::gpu::get_gpu_indvars< sdfg::cuda::ScheduleType_CUDA >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ get_gpu_indvars< sdfg::rocm::ScheduleType_ROCM >()

template symbolic::SymbolSet sdfg::gpu::get_gpu_indvars< sdfg::rocm::ScheduleType_ROCM >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ get_gpu_maps()

template<typename ScheduleT>

std::vector< structured_control_flow::Map * > sdfg::gpu::get_gpu_maps	(	structured_control_flow::Map &	node,
		analysis::AnalysisManager &	analysis_manager,
		GPUDimension	dimension )

Get all GPU Map nodes in a given dimension (in tree traversal order).

Unlike get_gpu_indvars, this preserves access to each Map's init / stride so the codegen can emit indvar = init + thread_flat_id * stride for arbitrary affine grid loops.

Template Parameters

ScheduleT Schedule type class with value() and dimension() static methods

Parameters

node	The current map node
analysis_manager	Analysis manager for loop analysis
dimension	GPU dimension (X, Y, or Z)

Returns: Vector of Map pointers in the given GPU dimension

◆ get_gpu_maps< sdfg::cuda::ScheduleType_CUDA >()

template std::vector< structured_control_flow::Map * > sdfg::gpu::get_gpu_maps< sdfg::cuda::ScheduleType_CUDA >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ get_gpu_maps< sdfg::rocm::ScheduleType_ROCM >()

template std::vector< structured_control_flow::Map * > sdfg::gpu::get_gpu_maps< sdfg::rocm::ScheduleType_ROCM >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	,
		GPUDimension	)

extern

◆ get_nested_level_maps()

void sdfg::gpu::get_nested_level_maps	(	structured_control_flow::StructuredLoop &	node,
		analysis::AnalysisManager &	analysis_manager,
		std::unordered_map< TargetLevel, structured_control_flow::StructuredLoop * > &	output )

◆ get_nested_schedule_types()

void sdfg::gpu::get_nested_schedule_types	(	structured_control_flow::StructuredLoop &	node,
		analysis::AnalysisManager &	analysis_manager,
		std::unordered_map< TargetLevel, structured_control_flow::ScheduleType > &	output )

◆ get_target_level_dim()

symbolic::Expression sdfg::gpu::get_target_level_dim	(	TargetLevel	target_level,
		int	warp_size )

◆ get_target_level_idx()

symbolic::Expression sdfg::gpu::get_target_level_idx ( TargetLevel target_level )

◆ gpu_block_size() [1/2]

symbolic::Integer sdfg::gpu::gpu_block_size ( const structured_control_flow::ScheduleType & schedule )

inline

Get the block size from any GPU schedule type Returns default values if not explicitly set:

X: 32 for CUDA, 64 for ROCM
Y: 8
Z: 4
Deprecated
This function is deprecated and will be removed in future versions. Use the new GPU schedule type classes instead.

◆ gpu_block_size() [2/2]

void sdfg::gpu::gpu_block_size	(	structured_control_flow::ScheduleType &	schedule,
		const symbolic::Expression	block_size )

inline

Set the block size on any GPU schedule type.

Deprecated: This function is deprecated and will be removed in future versions. Use the new GPU schedule type classes instead.

◆ gpu_dimension() [1/2]

GPUDimension sdfg::gpu::gpu_dimension ( const structured_control_flow::ScheduleType & schedule )

inline

Get the GPU dimension from any GPU schedule type.

Deprecated: This function is deprecated and will be removed in future versions. Use the new GPU schedule type classes instead.

◆ gpu_dimension() [2/2]

void sdfg::gpu::gpu_dimension	(	structured_control_flow::ScheduleType &	schedule,
		const GPUDimension &	dimension )

inline

Set the GPU dimension on any GPU schedule type.

Deprecated: This function is deprecated and will be removed in future versions. Use the new GPU schedule type classes instead.

◆ gpu_target_level() [1/2]

TargetLevel sdfg::gpu::gpu_target_level ( const structured_control_flow::ScheduleType & schedule )

inline

Get the GPU target level from any GPU schedule type.

◆ gpu_target_level() [2/2]

void sdfg::gpu::gpu_target_level	(	structured_control_flow::ScheduleType &	schedule,
		const TargetLevel &	target_level )

inline

Set the GPU target level on any GPU schedule type.

◆ gpu_warp_size() [1/2]

template<typename GPUType>

int64_t sdfg::gpu::gpu_warp_size ( )

◆ gpu_warp_size() [2/2]

int64_t sdfg::gpu::gpu_warp_size ( const structured_control_flow::ScheduleType & schedule )

◆ gpu_warp_size< sdfg::cuda::ScheduleType_CUDA_Offload >()

template<>

int64_t sdfg::gpu::gpu_warp_size< sdfg::cuda::ScheduleType_CUDA_Offload > ( )

◆ gpu_warp_size< sdfg::rocm::ScheduleType_ROCM_Offload >()

template<>

int64_t sdfg::gpu::gpu_warp_size< sdfg::rocm::ScheduleType_ROCM_Offload > ( )

◆ is_block_level()

bool sdfg::gpu::is_block_level ( TargetLevel target_level )

◆ is_gpu_schedule()

bool sdfg::gpu::is_gpu_schedule ( const structured_control_flow::ScheduleType & schedule )

Check if a schedule type is a GPU schedule (CUDA or ROCM).

◆ is_grid_level()

bool sdfg::gpu::is_grid_level ( TargetLevel target_level )

◆ is_outermost_gpu_map()

template<typename ScheduleT>

bool sdfg::gpu::is_outermost_gpu_map	(	structured_control_flow::Map &	node,
		analysis::AnalysisManager &	analysis_manager )

Check if a map is the outermost GPU map in the loop tree.

Template Parameters

ScheduleT Schedule type class with value() static method

Parameters

node	The current map node
analysis_manager	Analysis manager for loop analysis

Returns: true if this is the outermost GPU map, false otherwise

◆ is_outermost_gpu_map< sdfg::cuda::ScheduleType_CUDA >()

template bool sdfg::gpu::is_outermost_gpu_map< sdfg::cuda::ScheduleType_CUDA >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	)

extern

◆ is_outermost_gpu_map< sdfg::rocm::ScheduleType_ROCM >()

template bool sdfg::gpu::is_outermost_gpu_map< sdfg::rocm::ScheduleType_ROCM >	(	structured_control_flow::Map &	,
		analysis::AnalysisManager &	)

extern

◆ is_warp_level()

bool sdfg::gpu::is_warp_level ( TargetLevel target_level )

◆ LibraryNodeType_GpuMmaFill()

data_flow::LibraryNodeCode sdfg::gpu::LibraryNodeType_GpuMmaFill ( "gpu::GpuMmaFill" )

inline

◆ LibraryNodeType_GpuMmaFragmentEltwiseAdd()

data_flow::LibraryNodeCode sdfg::gpu::LibraryNodeType_GpuMmaFragmentEltwiseAdd ( "gpu::MmaFragmentEltwiseAdd" )

inline

◆ LibraryNodeType_GpuMmaFragmentLoad()

data_flow::LibraryNodeCode sdfg::gpu::LibraryNodeType_GpuMmaFragmentLoad ( "gpu::MmaFragmentLoad" )

inline

◆ LibraryNodeType_GpuMmaFragmentStore()

data_flow::LibraryNodeCode sdfg::gpu::LibraryNodeType_GpuMmaFragmentStore ( "gpu::MmaFragmentStore" )

inline

◆ LibraryNodeType_GpuMmaMatmul()

data_flow::LibraryNodeCode sdfg::gpu::LibraryNodeType_GpuMmaMatmul ( "gpu::GpuMmaMatmul" )

inline

◆ LibraryNodeType_GpuTileArrival()

data_flow::LibraryNodeCode sdfg::gpu::LibraryNodeType_GpuTileArrival ( "gpu::TileArrival" )

inline

◆ mma_fragment_layout_to_string()

const char * sdfg::gpu::mma_fragment_layout_to_string ( MmaFragmentLayout layout )

constexpr

◆ nested_parallelization_is_unsafe()

bool sdfg::gpu::nested_parallelization_is_unsafe	(	structured_control_flow::StructuredLoop &	loop,
		analysis::AnalysisManager &	analysis_manager )

Whether parallelizing loop to a new GPU grid dimension would race on a shared container.

Adding a grid dimension for loop folds the whole enclosing GPU kernel into a single flattened launch: loop 's iterations are distributed across the new dimension's threads, while every sibling of loop (and of its ancestors up to the outermost GPU map) is re-executed by every thread of that dimension. Since there is no grid-wide barrier inside a kernel, this is unsafe in two ways:

Producer/consumer across the fold: loop 's writes vary across the new dimension, so a replicated sibling that reads or writes one of those shared containers observes another thread's (incomplete) data (RAW/WAR/WAW). This is the reduction-accumulator -> consumer hazard, e.g. softmax where a reduce writes acc[i] and a following map divides by acc[i] in the same kernel.
Replicated self-accumulation: a sibling read-modify-write (acc[i] += x) is folded in once per replicated thread and races on the shared buffer.

Both hazards only apply to containers that escape the kernel (function arguments/externals or transients living outside, per ArgumentsAnalysis); a container confined to the kernel is privatized per thread and races nothing. For the self-accumulation hazard, a sibling that is itself a GPU map is exempt: it is already parallelized, so codegen maps it onto its own threads instead of replicating it.

This helper is schedule-agnostic (a map is "parallelized" iff its schedule is not Sequential), so it serves both the CUDA and ROCm nested-map transformations.

Parameters

loop	The sequential loop that a nested-parallelization transform wants to promote.
analysis_manager	Analysis manager (LoopAnalysis, Users, ArgumentsAnalysis).

Returns: true if folding loop into the kernel would introduce a data race.

◆ nested_warp_dim()

bool sdfg::gpu::nested_warp_dim	(	structured_control_flow::StructuredLoop &	loop,
		analysis::AnalysisManager &	analysis_manager )

◆ operator<<() [1/2]

std::ostream & sdfg::gpu::operator<<	(	std::ostream &	os,
		const GpuMmaFromMemoryLayout &	layout )

◆ operator<<() [2/2]

std::ostream & sdfg::gpu::operator<<	(	std::ostream &	os,
		const MmaBlockSize &	block_size )

◆ perfectly_nested_depth()

size_t sdfg::gpu::perfectly_nested_depth ( structured_control_flow::StructuredLoop * loop )

◆ reduce_strategy_from_string()

ReduceStrategy sdfg::gpu::reduce_strategy_from_string ( const std::string & value )

inline

◆ reduce_strategy_to_string()

std::string sdfg::gpu::reduce_strategy_to_string ( ReduceStrategy strategy )

inline

◆ serialize_mma_block_size()

void sdfg::gpu::serialize_mma_block_size	(	nlohmann::json &	j,
		const MmaBlockSize &	block_size )

◆ serialize_mma_from_memory_layout()

void sdfg::gpu::serialize_mma_from_memory_layout	(	nlohmann::json &	j,
		const GpuMmaFromMemoryLayout &	layout )

◆ target_level_from_string()

TargetLevel sdfg::gpu::target_level_from_string ( const std::string & value )

inline

◆ target_level_indvars()

symbolic::SymbolSet sdfg::gpu::target_level_indvars	(	structured_control_flow::StructuredLoop &	node,
		analysis::AnalysisManager &	analysis_manager,
		TargetLevel	target_level )

◆ to_string()

std::string sdfg::gpu::to_string ( const TargetLevel & target_level )

inline

◆ validate_wave_maps()

void sdfg::gpu::validate_wave_maps	(	structured_control_flow::StructuredLoop &	node,
		analysis::AnalysisManager &	analysis_manager,
		int64_t	warp_size )

Namespaces

Classes

Enumerations

Functions

Enumeration Type Documentation

◆ GPUDimension

◆ MmaFragmentLayout

◆ MmaFragmentType

◆ ReduceStrategy

◆ TargetLevel

Function Documentation

◆ analyze_nested_fold()

◆ default_reduce_strategy()

◆ deserialize_mma_block_size()

◆ deserialize_mma_from_memory_layout()

◆ emit_clipped_fragment_transfer()

◆ find_nested_gpu_blocksize()

◆ find_nested_gpu_blocksize< sdfg::cuda::ScheduleType_CUDA >()

◆ find_nested_gpu_blocksize< sdfg::rocm::ScheduleType_ROCM >()

◆ find_nested_gpu_iterations()

◆ find_nested_gpu_iterations< sdfg::cuda::ScheduleType_CUDA >()

◆ find_nested_gpu_iterations< sdfg::rocm::ScheduleType_ROCM >()

◆ find_x_block_owning_warp_level()

◆ get_gpu_indvars()

◆ get_gpu_indvars< sdfg::cuda::ScheduleType_CUDA >()

◆ get_gpu_indvars< sdfg::rocm::ScheduleType_ROCM >()

◆ get_gpu_maps()

◆ get_gpu_maps< sdfg::cuda::ScheduleType_CUDA >()

◆ get_gpu_maps< sdfg::rocm::ScheduleType_ROCM >()

◆ get_nested_level_maps()

◆ get_nested_schedule_types()

◆ get_target_level_dim()

◆ get_target_level_idx()

◆ gpu_block_size() [1/2]

◆ gpu_block_size() [2/2]

◆ gpu_dimension() [1/2]

◆ gpu_dimension() [2/2]

◆ gpu_target_level() [1/2]

◆ gpu_target_level() [2/2]

◆ gpu_warp_size() [1/2]

◆ gpu_warp_size() [2/2]

◆ gpu_warp_size< sdfg::cuda::ScheduleType_CUDA_Offload >()

◆ gpu_warp_size< sdfg::rocm::ScheduleType_ROCM_Offload >()

◆ is_block_level()

◆ is_gpu_schedule()

◆ is_grid_level()

◆ is_outermost_gpu_map()

◆ is_outermost_gpu_map< sdfg::cuda::ScheduleType_CUDA >()

◆ is_outermost_gpu_map< sdfg::rocm::ScheduleType_ROCM >()

◆ is_warp_level()

◆ LibraryNodeType_GpuMmaFill()

◆ LibraryNodeType_GpuMmaFragmentEltwiseAdd()

◆ LibraryNodeType_GpuMmaFragmentLoad()

◆ LibraryNodeType_GpuMmaFragmentStore()

◆ LibraryNodeType_GpuMmaMatmul()

◆ LibraryNodeType_GpuTileArrival()

◆ mma_fragment_layout_to_string()

◆ nested_parallelization_is_unsafe()

◆ nested_warp_dim()

◆ operator<<() [1/2]

◆ operator<<() [2/2]

◆ perfectly_nested_depth()

◆ reduce_strategy_from_string()

◆ reduce_strategy_to_string()

◆ serialize_mma_block_size()

◆ serialize_mma_from_memory_layout()

◆ target_level_from_string()

◆ target_level_indvars()

◆ to_string()

◆ validate_wave_maps()