m_cuda_poisson_fft

Uses

- cufftXt
- m_common
- m_poisson_fft
- m_cuda_spectral
- iso_c_binding
- iso_fortran_env
- cudafor
- m_mesh
- m_cuda_allocator
- m_cuda_sendrecv
- mpi
- m_tdsops
- m_field
- cufft
Help

Graph Key

Nodes of different colours represent the following:

Solid arrows point from a submodule to the (sub)module which it is descended from. Dashed arrows point from a module or program unit to modules which it uses.
Where possible, edges connecting nodes are given different colours to make them easier to distinguish in large graphs.

Used by

Help

Graph Key

Nodes of different colours represent the following:

Solid arrows point from a submodule to the (sub)module which it is descended from. Dashed arrows point from a module or program unit to modules which it uses.
Where possible, edges connecting nodes are given different colours to make them easier to distinguish in large graphs.

Interfaces

public interface cuda_poisson_fft_t

private function init(mesh, xdirps, ydirps, zdirps, lowmem) result(poisson_fft)

if stretching in y is 'bottom'

Arguments

Type	Intent	Optional	Attributes		Name
type(mesh_t),	intent(in),		target	::	mesh
type(dirps_t),	intent(in)			::	xdirps
type(dirps_t),	intent(in)			::	ydirps
type(dirps_t),	intent(in)			::	zdirps
logical,	intent(in),	optional		::	lowmem

Return Value type(cuda_poisson_fft_t)

interface

public function cufftExecR2C_C(plan, idata, odata) bind(C, name='cufftExecR2C')

Arguments

Type	Attributes		Name
integer(kind=c_int),	value	::	plan
type(c_ptr),	value	::	idata
type(c_ptr),	value	::	odata

Return Value integer(kind=c_int)

interface

public function cufftExecC2R_C(plan, idata, odata) bind(C, name='cufftExecC2R')

Arguments

Type	Attributes		Name
integer(kind=c_int),	value	::	plan
type(c_ptr),	value	::	idata
type(c_ptr),	value	::	odata

Return Value integer(kind=c_int)

Derived Types

type, public, extends(poisson_fft_t) :: cuda_poisson_fft_t

FFT based Poisson solver

Components

Type	Visibility	Attributes		Name		Initial
type(mesh_t),	public,	pointer	::	mesh	=>	null()
integer,	public		::	nx_glob			Global dimensions
integer,	public		::	ny_glob			Global dimensions
integer,	public		::	nz_glob			Global dimensions
integer,	public		::	nx_loc			Local dimensions
integer,	public		::	ny_loc			Local dimensions
integer,	public		::	nz_loc			Local dimensions
integer,	public		::	nx_perm			Local dimensions in the permuted slabs
integer,	public		::	ny_perm			Local dimensions in the permuted slabs
integer,	public		::	nz_perm			Local dimensions in the permuted slabs
integer,	public		::	nx_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	ny_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	nz_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	sp_st(3)			Offset per spectral dimension (dim1, dim2, dim3) in spectral space
complex(kind=dp),	public,	allocatable, dimension(:, :, :)	::	waves			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	allocatable, dimension(:)	::	ax			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bx			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	ay			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	by			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	az			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kx			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ky			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	exs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	eys			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ezs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2x			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2y			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2z			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_re			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_im			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_re			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_im			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_re			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_im			Staggared grid transformation
logical,	public		::	periodic_x			Periodicity in x, y, and z
logical,	public		::	periodic_y			Periodicity in x, y, and z
logical,	public		::	periodic_z			Periodicity in x, y, and z
logical,	public		::	stretched_y	=	.false.	Periodicity in x, y, and z
logical,	public		::	stretched_y_sym			Periodicity in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_im			Stretching operator matrices
logical,	public		::	lowmem	=	.false.	lowmem option, only used in CUDA backend
procedure(poisson_xxx),	public,	pointer	::	poisson	=>	null()	Procedure pointer to BC specific poisson solvers
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	waves_dev			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ax_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bx_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ay_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	by_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	az_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bz_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_odd_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_odd_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_even_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_even_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_odd_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_odd_im_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_even_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_even_im_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_im_dev			Stretching operator matrices
integer,	public		::	plan3D_fw			Forward and backward FFT transform plans
integer,	public		::	plan3D_bw			Forward and backward FFT transform plans
logical,	public		::	use_cufftmp	=	.true.	Flag to indicate whether cuFFTMp is used
logical,	public		::	is_000_case	=	.false.	Flag for cases
logical,	public		::	is_010_case	=	.false.
logical,	public		::	is_100_case	=	.false.
logical,	public		::	is_110_case	=	.false.
type(cudaLibXtDesc),	public,	pointer	::	xtdesc			cuFFTMp object manages decomposition and data storage
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_dev			Standard cuFFT storage
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev			cuFFT real workspace (input/output of R2C/C2R)
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev_110			Transposed real workspace for 110 case (nz, nx, ny)
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_mirror_dev			dim2 mirror of the local spectral slab, used by the 100 case to reach the paired split partner that lives on another rank
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_slab_send_dev			Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_send_dev			The single dim2 plane that falls outside the mirrored slab
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_recv_dev			The single dim2 plane that falls outside the mirrored slab
integer,	public		::	mirror_slab_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
integer,	public		::	mirror_plane_rank	=	0	Ranks holding the mirror slab and the extra mirror plane

Constructor

private function init (mesh, xdirps, ydirps, zdirps, lowmem)	if stretching in y is 'bottom'

Type-Bound Procedures

procedure, public :: base_init
procedure, public :: solve_poisson
procedure, public :: stretching_matrix
procedure, public :: waves_set
procedure, public :: get_km
procedure, public :: get_km_re
procedure, public :: get_km_im
procedure, public :: fft_forward => fft_forward_cuda
procedure, public :: fft_forward_010 => fft_forward_cuda
procedure, public :: fft_forward_100 => fft_forward_100_cuda
procedure, public :: fft_forward_110 => fft_forward_110_cuda
procedure, public :: fft_backward => fft_backward_cuda
procedure, public :: fft_backward_010 => fft_backward_cuda
procedure, public :: fft_backward_100 => fft_backward_100_cuda
procedure, public :: fft_backward_110 => fft_backward_110_cuda
procedure, public :: fft_postprocess_000 => fft_postprocess_000_cuda
procedure, public :: fft_postprocess_010 => fft_postprocess_010_cuda
procedure, public :: fft_postprocess_100 => fft_postprocess_100_cuda
procedure, public :: fft_postprocess_110 => fft_postprocess_110_cuda
procedure, public :: enforce_periodicity_x => enforce_periodicity_x_cuda
procedure, public :: undo_periodicity_x => undo_periodicity_x_cuda
procedure, public :: enforce_periodicity_y => enforce_periodicity_y_cuda
procedure, public :: undo_periodicity_y => undo_periodicity_y_cuda
procedure, public :: enforce_periodicity_xy => enforce_periodicity_xy_cuda
procedure, public :: undo_periodicity_xy => undo_periodicity_xy_cuda
procedure, public :: exchange_mirror_100

Subroutines

public subroutine fft_forward_110_cuda(self, f)

Forward FFT for 110 case: transpose (nx,ny,nz)->(nz,nx,ny) then R2C

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

public subroutine fft_backward_110_cuda(self, f)

Backward FFT for 110 case: C2R then transpose (nz,nx,ny)->(nx,ny,nz)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

public subroutine fft_forward_100_cuda(self, f)

Forward FFT for non-periodic-X case We transpose X<->Y so that the non-periodic direction becomes the "Y" direction in the transposed space, then use the same FFT approach as 010

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

public subroutine fft_backward_100_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

public subroutine fft_forward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

public subroutine fft_backward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

public subroutine fft_postprocess_000_cuda(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

public subroutine fft_postprocess_100_cuda(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

public subroutine exchange_mirror_100(self, c_dev)

Fills c_mirror_dev and c_plane_recv_dev with the dim2 mirror of the local spectral slab, so that the 100 paired split can read its partner mode without going looking for it in another rank's memory.

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
complex(kind=dp),	intent(in),		device, dimension(:, :, :)	::	c_dev

public subroutine fft_postprocess_010_cuda(self)

if stretching in y is 'bottom'

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

public subroutine fft_postprocess_110_cuda(self)

Spectral post-processing for 110 case (R2C Z-transpose) 7 separate kernel launches to avoid cross-block race conditions. Spectral array is (nz/2+1, nx, ny) = (nx_spec, ny_spec, nz_spec). Thread i->X (ny_spec), blockIdx%y k->Y (nz_spec), serial j->Z (nx_spec).

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

public subroutine enforce_periodicity_x_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine undo_periodicity_x_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine enforce_periodicity_y_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine undo_periodicity_y_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine enforce_periodicity_xy_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine undo_periodicity_xy_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

m_cuda_poisson_fft Module

Contents

Interfaces

Derived Types

Subroutines

Uses

Used by

Interfaces

public interface cuda_poisson_fft_t

private function init(mesh, xdirps, ydirps, zdirps, lowmem) result(poisson_fft)

Arguments

Return Value type(cuda_poisson_fft_t)

interface

public function cufftExecR2C_C(plan, idata, odata) bind(C, name='cufftExecR2C')

Arguments

Return Value integer(kind=c_int)

interface

public function cufftExecC2R_C(plan, idata, odata) bind(C, name='cufftExecC2R')

Arguments

Return Value integer(kind=c_int)

Derived Types

type, public, extends(poisson_fft_t) :: cuda_poisson_fft_t

Components

Constructor

Type-Bound Procedures

Subroutines

public subroutine fft_forward_110_cuda(self, f)

Arguments

public subroutine fft_backward_110_cuda(self, f)

Arguments

public subroutine fft_forward_100_cuda(self, f)

Arguments

public subroutine fft_backward_100_cuda(self, f)

Arguments

public subroutine fft_forward_cuda(self, f)

Arguments

public subroutine fft_backward_cuda(self, f)

Arguments

public subroutine fft_postprocess_000_cuda(self)

Arguments

public subroutine fft_postprocess_100_cuda(self)

Arguments

public subroutine exchange_mirror_100(self, c_dev)

Arguments

public subroutine fft_postprocess_010_cuda(self)

Arguments

public subroutine fft_postprocess_110_cuda(self)

Arguments

public subroutine enforce_periodicity_x_cuda(self, f_out, f_in)

Arguments

public subroutine undo_periodicity_x_cuda(self, f_out, f_in)

Arguments

public subroutine enforce_periodicity_y_cuda(self, f_out, f_in)

Arguments

public subroutine undo_periodicity_y_cuda(self, f_out, f_in)

Arguments

public subroutine enforce_periodicity_xy_cuda(self, f_out, f_in)

Arguments

public subroutine undo_periodicity_xy_cuda(self, f_out, f_in)

Arguments