cpid::distributed Namespace Reference

Classes
class	Context
class	Work

Typedefs
template<typename T >
using	IsTorchDType = typename std::enable_if_t< std::is_same< T, uint8_t >::value||std::is_same< T, char >::value||std::is_same< T, int8_t >::value||std::is_same< T, int16_t >::value||std::is_same< T, int32_t >::value||std::is_same< T, int64_t >::value||std::is_same< T, float >::value||std::is_same< T, double >::value >

Functions
void	init ()
void	setGPUToLocalRank ()
	Sets CUDA device to the local (if available) or MPI rank, both modulo the number of available devices. More...
std::shared_ptr< Context >	globalContext ()
	FOR_ALL_TYPES (ALLREDUCE)
template<typename T , IsTorchDType< T > * >
Work	allreduce (T *ptr, int64_t s, ReduceOp op)
template<typename T , IsTorchDType< T > * >
Work	allreduce (std::vector< T > &v, ReduceOp op)
Work	allreduce (torch::Tensor x, ReduceOp op)
Work	allreduceGradients (ag::Container const &x, ReduceOp op)
template<typename T , IsTorchDType< T > * >
Work	broadcast (T *ptr, int64_t s, int root)
template<typename T , IsTorchDType< T > * >
Work	broadcast (std::vector< T > &v, int root)
Work	broadcast (torch::Tensor x, int root)
Work	broadcast (ag::Container const &x, int root)
template<typename T , IsTorchDType< T > * >
Work	allgather (T *out, T *in, int64_t s)
template<typename T , IsTorchDType< T > * >
Work	allgather (T *out, torch::Tensor in)
Work	allgather (torch::Tensor out, torch::Tensor in)
Work	barrier ()
	FOR_ALL_TYPES (FORCE_INSTANTIATION)

Typedef Documentation

template<typename T >

using cpid::distributed::IsTorchDType = typedef typename std::enable_if_t< std::is_same<T, uint8_t>::value || std::is_same<T, char>::value || std::is_same<T, int8_t>::value || std::is_same<T, int16_t>::value || std::is_same<T, int32_t>::value || std::is_same<T, int64_t>::value || std::is_same<T, float>::value || std::is_same<T, double>::value>

Function Documentation

template<typename T , IsTorchDType< T > * >

Work cpid::distributed::allgather	(	T *	out,
		T *	in,
		int64_t	s
	)

template<typename T , IsTorchDType< T > * >

Work cpid::distributed::allgather	(	T *	out,
		torch::Tensor	in
	)

Work cpid::distributed::allgather	(	torch::Tensor	out,
		torch::Tensor	in
	)

template<typename T , IsTorchDType< T > * >

Work cpid::distributed::allreduce	(	T *	ptr,
		int64_t	s,
		ReduceOp	op
	)

template<typename T , IsTorchDType< T > * >

Work cpid::distributed::allreduce	(	std::vector< T > &	v,
		ReduceOp	op
	)

Work cpid::distributed::allreduce	(	torch::Tensor	x,
		ReduceOp	op
	)

Work cpid::distributed::allreduceGradients	(	ag::Container const &	x,
		ReduceOp	op
	)

Work cpid::distributed::barrier

(

)

template<typename T , IsTorchDType< T > * >

Work cpid::distributed::broadcast	(	T *	ptr,
		int64_t	s,
		int	root
	)

template<typename T , IsTorchDType< T > * >

Work cpid::distributed::broadcast	(	std::vector< T > &	v,
		int	root
	)

Work cpid::distributed::broadcast	(	torch::Tensor	x,
		int	root
	)

Work cpid::distributed::broadcast	(	ag::Container const &	x,
		int	root
	)

cpid::distributed::FOR_ALL_TYPES

(

ALLREDUCE

)

cpid::distributed::FOR_ALL_TYPES

(

FORCE_INSTANTIATION

)

std::shared_ptr< Context > cpid::distributed::globalContext

(

)

void cpid::distributed::init

(

)

void cpid::distributed::setGPUToLocalRank

(

)

Sets CUDA device to the local (if available) or MPI rank, both modulo the number of available devices.

Does nothing when no CUDA is avail. init() calls setGPUToLocalRank() already, but since the result is thread-local it's necessary to call it from any thread that is spawned