m_cuda_poisson_fft

create_fft_plan is deliberately public: the static memory estimator (m_cuda_memcheck_device, src/backend/cuda/memcheck_device.f90) calls it standalone (plan only, no mesh/case/fields) to capture the real cuFFT/cuFFTMp workspace size via its worksize out-argument, instead of duplicating cufftMakePlan3D's call sequence a second time.

Uses

- m_cuda_spectral
- iso_c_binding
- cufftXt
- m_mesh
- m_cuda_sendrecv
- m_mpi
- m_cuda_errors
- m_field
- m_poisson_fft
- m_cuda_allocator
- m_logger
- m_tdsops
- cudafor
- cufft
- m_common
Help

Graph Key

Nodes of different colours represent the following:

Solid arrows point from a submodule to the (sub)module which it is descended from. Dashed arrows point from a module or program unit to modules which it uses.
Where possible, edges connecting nodes are given different colours to make them easier to distinguish in large graphs.

Used by

Help

Graph Key

Nodes of different colours represent the following:

Solid arrows point from a submodule to the (sub)module which it is descended from. Dashed arrows point from a module or program unit to modules which it uses.
Where possible, edges connecting nodes are given different colours to make them easier to distinguish in large graphs.

Interfaces

interface

public function cufftExecR2C_C(plan, idata, odata) bind(C, name='cufftExecR2C')

Arguments

Type	Attributes		Name
integer(kind=c_int),	value	::	plan
type(c_ptr),	value	::	idata
type(c_ptr),	value	::	odata

Return Value integer(kind=c_int)

interface

public function cufftExecC2R_C(plan, idata, odata) bind(C, name='cufftExecC2R')

Arguments

Type	Attributes		Name
integer(kind=c_int),	value	::	plan
type(c_ptr),	value	::	idata
type(c_ptr),	value	::	odata

Return Value integer(kind=c_int)

Derived Types

type, public, abstract, extends(poisson_fft_t) :: cuda_poisson_fft_t

FFT based Poisson solver, the state and the plain transform that every boundary condition case shares. Each case extends this type with its own buffers, see cuda_poisson_000_t to cuda_poisson_110_t.

Components

Type	Visibility	Attributes		Name		Initial
type(mesh_t),	public,	pointer	::	mesh	=>	null()
integer,	public		::	nx_glob			Global dimensions
integer,	public		::	ny_glob			Global dimensions
integer,	public		::	nz_glob			Global dimensions
integer,	public		::	nx_loc			Local dimensions
integer,	public		::	ny_loc			Local dimensions
integer,	public		::	nz_loc			Local dimensions
integer,	public		::	nx_perm			Local dimensions in the permuted slabs
integer,	public		::	ny_perm			Local dimensions in the permuted slabs
integer,	public		::	nz_perm			Local dimensions in the permuted slabs
integer,	public		::	nx_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	ny_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	nz_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	sp_st(3)			Offset per spectral dimension (dim1, dim2, dim3) in spectral space
complex(kind=dp),	public,	allocatable, dimension(:, :, :)	::	waves			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	allocatable, dimension(:)	::	ax			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bx			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	ay			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	by			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	az			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kx			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ky			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	exs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	eys			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ezs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2x			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2y			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2z			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_im			Staggered grid transformation
logical,	public		::	periodic_x			Periodicity in x, y, and z
logical,	public		::	periodic_y			Periodicity in x, y, and z
logical,	public		::	periodic_z			Periodicity in x, y, and z
logical,	public		::	stretched_y	=	.false.	Periodicity in x, y, and z
logical,	public		::	stretched_y_sym			Periodicity in x, y, and z
character(len=3),	public		::	bc_code			Boundary condition case, see poisson_bc_code. The backend's init sets it first, ahead of base_init.
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_im			Stretching operator matrices
logical,	public		::	lowmem	=	.false.	lowmem option, only used in CUDA backend
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	waves_dev			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ax_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bx_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ay_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	by_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	az_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bz_dev			Wave numbers in x, y, and z
integer,	public		::	plan3D_fw			Forward and backward FFT transform plans
integer,	public		::	plan3D_bw			Forward and backward FFT transform plans
logical,	public		::	use_cufftmp	=	.true.	Flag to indicate whether cuFFTMp is used
type(cudaLibXtDesc),	public,	pointer	::	xtdesc			cuFFTMp object manages decomposition and data storage
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_dev			Standard cuFFT storage
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev			cuFFT real workspace (input/output of R2C/C2R)
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_mirror_dev			dim2 mirror of the local spectral slab, used by the 010, 100 and 110 cases to reach the paired split partner that lives on another rank. Only init_dim2_mirror allocates it, and init_cuda reads its allocation status as "this case needs cuFFTMp".
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_slab_send_dev			Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_send_dev			The single dim2 plane that falls outside the mirrored slab
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_recv_dev			The single dim2 plane that falls outside the mirrored slab
integer,	public		::	mirror_slab_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
integer,	public		::	mirror_plane_rank	=	0	Ranks holding the mirror slab and the extra mirror plane

Type-Bound Procedures

procedure(fft_postprocess), public, deferred :: fft_postprocess
procedure, public :: enforce_periodicity
procedure, public :: undo_periodicity
procedure, public :: base_init
procedure, public :: solve_poisson
procedure, public :: stretching_matrix
procedure, public :: waves_set
procedure, public :: get_km
procedure, public :: get_km_re
procedure, public :: get_km_im
procedure, public :: init => init_cuda
procedure, public :: init_case
procedure, public :: create_plans
procedure, public :: init_dim2_mirror
procedure, public :: fft_forward => fft_forward_cuda
procedure, public :: fft_backward => fft_backward_cuda
procedure, public :: exchange_dim2_mirror

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_000_t

Periodic in x, y and z

Components

Type	Visibility	Attributes		Name		Initial
type(mesh_t),	public,	pointer	::	mesh	=>	null()
integer,	public		::	nx_glob			Global dimensions
integer,	public		::	ny_glob			Global dimensions
integer,	public		::	nz_glob			Global dimensions
integer,	public		::	nx_loc			Local dimensions
integer,	public		::	ny_loc			Local dimensions
integer,	public		::	nz_loc			Local dimensions
integer,	public		::	nx_perm			Local dimensions in the permuted slabs
integer,	public		::	ny_perm			Local dimensions in the permuted slabs
integer,	public		::	nz_perm			Local dimensions in the permuted slabs
integer,	public		::	nx_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	ny_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	nz_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	sp_st(3)			Offset per spectral dimension (dim1, dim2, dim3) in spectral space
complex(kind=dp),	public,	allocatable, dimension(:, :, :)	::	waves			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	allocatable, dimension(:)	::	ax			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bx			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	ay			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	by			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	az			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kx			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ky			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	exs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	eys			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ezs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2x			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2y			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2z			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_im			Staggered grid transformation
logical,	public		::	periodic_x			Periodicity in x, y, and z
logical,	public		::	periodic_y			Periodicity in x, y, and z
logical,	public		::	periodic_z			Periodicity in x, y, and z
logical,	public		::	stretched_y	=	.false.	Periodicity in x, y, and z
logical,	public		::	stretched_y_sym			Periodicity in x, y, and z
character(len=3),	public		::	bc_code			Boundary condition case, see poisson_bc_code. The backend's init sets it first, ahead of base_init.
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_im			Stretching operator matrices
logical,	public		::	lowmem	=	.false.	lowmem option, only used in CUDA backend
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	waves_dev			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ax_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bx_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ay_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	by_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	az_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bz_dev			Wave numbers in x, y, and z
integer,	public		::	plan3D_fw			Forward and backward FFT transform plans
integer,	public		::	plan3D_bw			Forward and backward FFT transform plans
logical,	public		::	use_cufftmp	=	.true.	Flag to indicate whether cuFFTMp is used
type(cudaLibXtDesc),	public,	pointer	::	xtdesc			cuFFTMp object manages decomposition and data storage
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_dev			Standard cuFFT storage
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev			cuFFT real workspace (input/output of R2C/C2R)
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_mirror_dev			dim2 mirror of the local spectral slab, used by the 010, 100 and 110 cases to reach the paired split partner that lives on another rank. Only init_dim2_mirror allocates it, and init_cuda reads its allocation status as "this case needs cuFFTMp".
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_slab_send_dev			Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_send_dev			The single dim2 plane that falls outside the mirrored slab
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_recv_dev			The single dim2 plane that falls outside the mirrored slab
integer,	public		::	mirror_slab_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
integer,	public		::	mirror_plane_rank	=	0	Ranks holding the mirror slab and the extra mirror plane

Type-Bound Procedures

procedure, public :: enforce_periodicity
procedure, public :: undo_periodicity
procedure, public :: base_init
procedure, public :: stretching_matrix
procedure, public :: waves_set
procedure, public :: get_km
procedure, public :: get_km_re
procedure, public :: get_km_im
procedure, public :: init => init_cuda
procedure, public :: init_case
procedure, public :: create_plans
procedure, public :: init_dim2_mirror
procedure, public :: fft_forward => fft_forward_cuda
procedure, public :: fft_backward => fft_backward_cuda
procedure, public :: exchange_dim2_mirror
procedure, public :: solve_poisson => solve_poisson_000_cuda
procedure, public :: fft_postprocess => fft_postprocess_000_cuda

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_010_t

Periodic in x and z, non-periodic in y, the only case that supports stretching in y

Components

Type	Visibility	Attributes		Name		Initial
type(mesh_t),	public,	pointer	::	mesh	=>	null()
integer,	public		::	nx_glob			Global dimensions
integer,	public		::	ny_glob			Global dimensions
integer,	public		::	nz_glob			Global dimensions
integer,	public		::	nx_loc			Local dimensions
integer,	public		::	ny_loc			Local dimensions
integer,	public		::	nz_loc			Local dimensions
integer,	public		::	nx_perm			Local dimensions in the permuted slabs
integer,	public		::	ny_perm			Local dimensions in the permuted slabs
integer,	public		::	nz_perm			Local dimensions in the permuted slabs
integer,	public		::	nx_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	ny_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	nz_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	sp_st(3)			Offset per spectral dimension (dim1, dim2, dim3) in spectral space
complex(kind=dp),	public,	allocatable, dimension(:, :, :)	::	waves			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	allocatable, dimension(:)	::	ax			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bx			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	ay			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	by			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	az			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kx			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ky			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	exs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	eys			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ezs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2x			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2y			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2z			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_im			Staggered grid transformation
logical,	public		::	periodic_x			Periodicity in x, y, and z
logical,	public		::	periodic_y			Periodicity in x, y, and z
logical,	public		::	periodic_z			Periodicity in x, y, and z
logical,	public		::	stretched_y	=	.false.	Periodicity in x, y, and z
logical,	public		::	stretched_y_sym			Periodicity in x, y, and z
character(len=3),	public		::	bc_code			Boundary condition case, see poisson_bc_code. The backend's init sets it first, ahead of base_init.
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_im			Stretching operator matrices
logical,	public		::	lowmem	=	.false.	lowmem option, only used in CUDA backend
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	waves_dev			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ax_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bx_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ay_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	by_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	az_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bz_dev			Wave numbers in x, y, and z
integer,	public		::	plan3D_fw			Forward and backward FFT transform plans
integer,	public		::	plan3D_bw			Forward and backward FFT transform plans
logical,	public		::	use_cufftmp	=	.true.	Flag to indicate whether cuFFTMp is used
type(cudaLibXtDesc),	public,	pointer	::	xtdesc			cuFFTMp object manages decomposition and data storage
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_dev			Standard cuFFT storage
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev			cuFFT real workspace (input/output of R2C/C2R)
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_mirror_dev			dim2 mirror of the local spectral slab, used by the 010, 100 and 110 cases to reach the paired split partner that lives on another rank. Only init_dim2_mirror allocates it, and init_cuda reads its allocation status as "this case needs cuFFTMp".
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_slab_send_dev			Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_send_dev			The single dim2 plane that falls outside the mirrored slab
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_recv_dev			The single dim2 plane that falls outside the mirrored slab
integer,	public		::	mirror_slab_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
integer,	public		::	mirror_plane_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_odd_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_odd_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_even_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_even_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_odd_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_odd_im_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_even_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_even_im_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_im_dev			Stretching operator matrices

Type-Bound Procedures

procedure, public :: base_init
procedure, public :: solve_poisson
procedure, public :: stretching_matrix
procedure, public :: waves_set
procedure, public :: get_km
procedure, public :: get_km_re
procedure, public :: get_km_im
procedure, public :: init => init_cuda
procedure, public :: create_plans
procedure, public :: init_dim2_mirror
procedure, public :: fft_forward => fft_forward_cuda
procedure, public :: fft_backward => fft_backward_cuda
procedure, public :: exchange_dim2_mirror
procedure, public :: init_case => init_case_010
procedure, public :: fft_postprocess => fft_postprocess_010_cuda
procedure, public :: enforce_periodicity => enforce_periodicity_y_cuda
procedure, public :: undo_periodicity => undo_periodicity_y_cuda

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_100_t

Non-periodic in x, periodic in y and z

Components

Type	Visibility	Attributes		Name		Initial
type(mesh_t),	public,	pointer	::	mesh	=>	null()
integer,	public		::	nx_glob			Global dimensions
integer,	public		::	ny_glob			Global dimensions
integer,	public		::	nz_glob			Global dimensions
integer,	public		::	nx_loc			Local dimensions
integer,	public		::	ny_loc			Local dimensions
integer,	public		::	nz_loc			Local dimensions
integer,	public		::	nx_perm			Local dimensions in the permuted slabs
integer,	public		::	ny_perm			Local dimensions in the permuted slabs
integer,	public		::	nz_perm			Local dimensions in the permuted slabs
integer,	public		::	nx_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	ny_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	nz_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	sp_st(3)			Offset per spectral dimension (dim1, dim2, dim3) in spectral space
complex(kind=dp),	public,	allocatable, dimension(:, :, :)	::	waves			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	allocatable, dimension(:)	::	ax			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bx			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	ay			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	by			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	az			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kx			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ky			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	exs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	eys			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ezs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2x			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2y			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2z			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_im			Staggered grid transformation
logical,	public		::	periodic_x			Periodicity in x, y, and z
logical,	public		::	periodic_y			Periodicity in x, y, and z
logical,	public		::	periodic_z			Periodicity in x, y, and z
logical,	public		::	stretched_y	=	.false.	Periodicity in x, y, and z
logical,	public		::	stretched_y_sym			Periodicity in x, y, and z
character(len=3),	public		::	bc_code			Boundary condition case, see poisson_bc_code. The backend's init sets it first, ahead of base_init.
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_im			Stretching operator matrices
logical,	public		::	lowmem	=	.false.	lowmem option, only used in CUDA backend
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	waves_dev			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ax_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bx_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ay_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	by_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	az_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bz_dev			Wave numbers in x, y, and z
integer,	public		::	plan3D_fw			Forward and backward FFT transform plans
integer,	public		::	plan3D_bw			Forward and backward FFT transform plans
logical,	public		::	use_cufftmp	=	.true.	Flag to indicate whether cuFFTMp is used
type(cudaLibXtDesc),	public,	pointer	::	xtdesc			cuFFTMp object manages decomposition and data storage
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_dev			Standard cuFFT storage
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev			cuFFT real workspace (input/output of R2C/C2R)
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_mirror_dev			dim2 mirror of the local spectral slab, used by the 010, 100 and 110 cases to reach the paired split partner that lives on another rank. Only init_dim2_mirror allocates it, and init_cuda reads its allocation status as "this case needs cuFFTMp".
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_slab_send_dev			Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_send_dev			The single dim2 plane that falls outside the mirrored slab
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_recv_dev			The single dim2 plane that falls outside the mirrored slab
integer,	public		::	mirror_slab_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
integer,	public		::	mirror_plane_rank	=	0	Ranks holding the mirror slab and the extra mirror plane

Type-Bound Procedures

procedure, public :: base_init
procedure, public :: solve_poisson
procedure, public :: stretching_matrix
procedure, public :: waves_set
procedure, public :: get_km
procedure, public :: get_km_re
procedure, public :: get_km_im
procedure, public :: init => init_cuda
procedure, public :: create_plans
procedure, public :: init_dim2_mirror
procedure, public :: exchange_dim2_mirror
procedure, public :: init_case => init_case_100
procedure, public :: fft_forward => fft_forward_100_cuda
procedure, public :: fft_backward => fft_backward_100_cuda
procedure, public :: fft_postprocess => fft_postprocess_100_cuda
procedure, public :: enforce_periodicity => enforce_periodicity_x_cuda
procedure, public :: undo_periodicity => undo_periodicity_x_cuda

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_110_t

Non-periodic in x and y, periodic in z

Components

Type	Visibility	Attributes		Name		Initial
type(mesh_t),	public,	pointer	::	mesh	=>	null()
integer,	public		::	nx_glob			Global dimensions
integer,	public		::	ny_glob			Global dimensions
integer,	public		::	nz_glob			Global dimensions
integer,	public		::	nx_loc			Local dimensions
integer,	public		::	ny_loc			Local dimensions
integer,	public		::	nz_loc			Local dimensions
integer,	public		::	nx_perm			Local dimensions in the permuted slabs
integer,	public		::	ny_perm			Local dimensions in the permuted slabs
integer,	public		::	nz_perm			Local dimensions in the permuted slabs
integer,	public		::	nx_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	ny_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	nz_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	sp_st(3)			Offset per spectral dimension (dim1, dim2, dim3) in spectral space
complex(kind=dp),	public,	allocatable, dimension(:, :, :)	::	waves			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	allocatable, dimension(:)	::	ax			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bx			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	ay			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	by			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	az			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kx			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ky			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	exs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	eys			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ezs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2x			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2y			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2z			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_im			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_re			Staggered grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_im			Staggered grid transformation
logical,	public		::	periodic_x			Periodicity in x, y, and z
logical,	public		::	periodic_y			Periodicity in x, y, and z
logical,	public		::	periodic_z			Periodicity in x, y, and z
logical,	public		::	stretched_y	=	.false.	Periodicity in x, y, and z
logical,	public		::	stretched_y_sym			Periodicity in x, y, and z
character(len=3),	public		::	bc_code			Boundary condition case, see poisson_bc_code. The backend's init sets it first, ahead of base_init.
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_im			Stretching operator matrices
logical,	public		::	lowmem	=	.false.	lowmem option, only used in CUDA backend
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	waves_dev			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ax_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bx_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ay_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	by_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	az_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bz_dev			Wave numbers in x, y, and z
integer,	public		::	plan3D_fw			Forward and backward FFT transform plans
integer,	public		::	plan3D_bw			Forward and backward FFT transform plans
logical,	public		::	use_cufftmp	=	.true.	Flag to indicate whether cuFFTMp is used
type(cudaLibXtDesc),	public,	pointer	::	xtdesc			cuFFTMp object manages decomposition and data storage
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_dev			Standard cuFFT storage
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev			cuFFT real workspace (input/output of R2C/C2R)
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_mirror_dev			dim2 mirror of the local spectral slab, used by the 010, 100 and 110 cases to reach the paired split partner that lives on another rank. Only init_dim2_mirror allocates it, and init_cuda reads its allocation status as "this case needs cuFFTMp".
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_slab_send_dev			Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_send_dev			The single dim2 plane that falls outside the mirrored slab
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_recv_dev			The single dim2 plane that falls outside the mirrored slab
integer,	public		::	mirror_slab_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
integer,	public		::	mirror_plane_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev_110			Transposed real workspace (nz, nx, ny), single rank only
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	r_a2a_send_dev			All-to-all buffers redistributing the field between the physical Z-slab (nx, ny, nz_l) and the cuFFTMp Y-slab (2*(nz/2+1), nx, ny_l), sized (nz_l, nx, ny_l, nproc). Ordinary device memory, for the same reason as c_slab_send_dev.
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	r_a2a_recv_dev			All-to-all buffers redistributing the field between the physical Z-slab (nx, ny, nz_l) and the cuFFTMp Y-slab (2*(nz/2+1), nx, ny_l), sized (nz_l, nx, ny_l, nproc). Ordinary device memory, for the same reason as c_slab_send_dev.

Type-Bound Procedures

procedure, public :: base_init
procedure, public :: solve_poisson
procedure, public :: stretching_matrix
procedure, public :: waves_set
procedure, public :: get_km
procedure, public :: get_km_re
procedure, public :: get_km_im
procedure, public :: init => init_cuda
procedure, public :: init_dim2_mirror
procedure, public :: exchange_dim2_mirror
procedure, public :: init_case => init_case_110
procedure, public :: create_plans => create_plans_110
procedure, public :: fft_forward => fft_forward_110_cuda
procedure, public :: fft_backward => fft_backward_110_cuda
procedure, public :: fft_postprocess => fft_postprocess_110_cuda
procedure, public :: enforce_periodicity => enforce_periodicity_xy_cuda
procedure, public :: undo_periodicity => undo_periodicity_xy_cuda
procedure, public :: redistribute_110_to_yslab
procedure, public :: redistribute_110_to_zslab

Functions

public pure function cuda_fft_decomp_reason(bc_code, cdims, nproc_dir) result(reason)

The reason the solver cannot run the BC class bc_code on the cell dimensions cdims and the decomposition nproc_dir, blank when it can. The text is the message the solver init aborts with. Shared by the solver init and x3d2-memcheck.

Arguments

Type	Intent		Name
integer,	intent(in)	::	bc_code
integer,	intent(in)	::	cdims(3)
integer,	intent(in)	::	nproc_dir(3)

Return Value character(len=110)

Subroutines

public subroutine create_fft_plan(plan, use_cufftmp, nx, ny, nz, plan_type, plan_name, worksize_out, stat, auto_alloc_off)

Helper subroutine to create FFT plan with automatic cuFFTMp fallback

Arguments

Type	Intent	Optional		Name
integer,	intent(inout)		::	plan
logical,	intent(inout)		::	use_cufftmp
integer,	intent(in)		::	nx
integer,	intent(in)		::	ny
integer,	intent(in)		::	nz
integer,	intent(in)		::	plan_type
character(len=*),	intent(in)		::	plan_name
integer(kind=int_ptr_kind()),	intent(out),	optional	::	worksize_out	Set to cufftMakePlan3D's reported workspace size (bytes) when present. Optional and unused by every real solver call site (init, below); added only so the static memory estimator can read the exact FFT workspace size without a second, duplicate cufftMakePlan3D call.
integer,	intent(out),	optional	::	stat	When present, a failed plain cuFFT plan creation returns its cuFFT error code here instead of stopping. Used only by the static memory estimator; the solver call sites leave it out and keep the stop.
logical,	intent(in),	optional	::	auto_alloc_off	When present and true, a cuFFTMp plan is made with auto-allocation off. The plan then reports its workspace size in worksize_out and does not allocate the workspace. Used only by the static memory estimator.

public subroutine create_fft_plan_pair(plan_fw, plan_bw, use_cufftmp, fft_n, fw_type, bw_type, label, worksize_fw, worksize_bw, stat, no_auto_alloc)

Creates the forward and the backward FFT plan the way the solver does. A backward plan that forces the fallback to plain cuFFT makes the forward plan, built on cuFFTMp, get rebuilt on plain cuFFT, so both plans end on the same library. Shared by the solver init and the x3d2-memcheck probe. The optional arguments are the ones of create_fft_plan. label is appended to the plan names in the messages. When stat is present and not zero, the plans made so far are destroyed and the other plan is left undefined.

Arguments

Type	Intent	Optional		Name
integer,	intent(inout)		::	plan_fw
integer,	intent(inout)		::	plan_bw
logical,	intent(inout)		::	use_cufftmp
integer,	intent(in)		::	fft_n(3)
integer,	intent(in)		::	fw_type
integer,	intent(in)		::	bw_type
character(len=*),	intent(in)		::	label
integer(kind=int_ptr_kind()),	intent(out),	optional	::	worksize_fw
integer(kind=int_ptr_kind()),	intent(out),	optional	::	worksize_bw
integer,	intent(out),	optional	::	stat
logical,	intent(in),	optional	::	no_auto_alloc

public pure subroutine cuda_fft_spectral_dims(bc_code, dims_glob, dims_loc, nproc_dir, nrank_dir, n_spec, n_sp_st)

The size n_spec and the offset n_sp_st of the local spectral pencil in the permuted state, for the BC class bc_code on the cell dimensions dims_glob and dims_loc, the decomposition nproc_dir and the rank position nrank_dir. A BC class the solver does not support gives zeros. Shared by the solver init and the x3d2-memcheck estimate.

Arguments

Type	Intent		Name
integer,	intent(in)	::	bc_code
integer,	intent(in)	::	dims_glob(3)
integer,	intent(in)	::	dims_loc(3)
integer,	intent(in)	::	nproc_dir(3)
integer,	intent(in)	::	nrank_dir(3)
integer,	intent(out)	::	n_spec(3)
integer,	intent(out)	::	n_sp_st(3)

public pure subroutine cuda_fft_plan_setup(bc_code, dims_glob, dims_loc, fft_n, fft_n_loc, fw_type, bw_type)

The cuFFT plan dimensions and types of a BC class. The 100 case transposes X<->Y and runs the FFT on (ny, nx, nz), the 110 case transposes to (nz, nx, ny) with R2C along Z, and the 000 and 010 cases run it on (nx, ny, nz). fft_n holds the global plan dimensions and fft_n_loc the local extents of the first two. Shared by the solver init and the x3d2-memcheck probe.

Arguments

Type	Intent		Name
integer,	intent(in)	::	bc_code
integer,	intent(in)	::	dims_glob(3)
integer,	intent(in)	::	dims_loc(3)
integer,	intent(out)	::	fft_n(3)
integer,	intent(out)	::	fft_n_loc(2)
integer,	intent(out)	::	fw_type
integer,	intent(out)	::	bw_type

public subroutine create_cuda_poisson_fft(poisson_fft, mesh, xdirps, ydirps, zdirps, lowmem)

Allocates the solver type of the mesh's boundary condition case and initialises it.

Arguments

Type	Intent	Optional	Attributes		Name
class(poisson_fft_t),	intent(out),		pointer	::	poisson_fft
type(mesh_t),	intent(in),		target	::	mesh
type(dirps_t),	intent(in)			::	xdirps
type(dirps_t),	intent(in)			::	ydirps
type(dirps_t),	intent(in)			::	zdirps
logical,	intent(in),	optional		::	lowmem

public subroutine init_case(self, mesh)

Sets up the device state of the case ahead of the FFT plans. The 000 case needs nothing beyond the shared state.

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
type(mesh_t),	intent(in)			::	mesh

public subroutine init_case_010(self, mesh)

if stretching in y is 'bottom'

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_010_t)				::	self
type(mesh_t),	intent(in)			::	mesh

public subroutine init_case_100(self, mesh)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_100_t)				::	self
type(mesh_t),	intent(in)			::	mesh

public subroutine init_case_110(self, mesh)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_110_t)				::	self
type(mesh_t),	intent(in)			::	mesh

public subroutine init_dim2_mirror(self, mesh)

The 010, 100 and 110 cases pair each dim2 spectral mode with its mirror, which cuFFTMp may place on another rank. Slab index r of P owns global dim2 modes [rm + 1, (r + 1)m] with m = ny_spec, and the mirror of that range is slab P - 1 - r shifted up by one plane, so the extra plane comes from slab P - r. Both partners are symmetric. A single rank holds every partner itself and allocates nothing. This is the only place c_mirror_dev is allocated, see the cuFFTMp check in init_cuda.

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
type(mesh_t),	intent(in)			::	mesh

public subroutine create_plans(self, mesh, fft_n, fft_n_loc, fw_plan_type, bw_plan_type)

Standard R2C/C2R plans on the grid permuted into the spectral layout, with cuFFTMp fallback. This serves the 000, 010 and 100 cases. fft_n and fft_n_loc are the plan dimensions from cuda_fft_plan_setup.

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
type(mesh_t),	intent(in)	::	mesh
integer,	intent(in)	::	fft_n(3)
integer,	intent(in)	::	fft_n_loc(2)
integer,	intent(in)	::	fw_plan_type
integer,	intent(in)	::	bw_plan_type

public subroutine create_plans_110(self, mesh, fft_n, fft_n_loc, fw_plan_type, bw_plan_type)

R2C with Z-transpose, on (nz, nx, ny). A single rank keeps the plain cuFFT path; on multiple ranks cuFFTMp distributes the transform, and the physical Z-slab is redistributed into its Y-slab input through an MPI_Alltoall (see redistribute_110_to_yslab).

Arguments

Type	Intent		Name
class(cuda_poisson_110_t)		::	self
type(mesh_t),	intent(in)	::	mesh
integer,	intent(in)	::	fft_n(3)
integer,	intent(in)	::	fft_n_loc(2)
integer,	intent(in)	::	fw_plan_type
integer,	intent(in)	::	bw_plan_type

public subroutine solve_poisson_000_cuda(self, f, temp)

The 000 case has nothing to reorder, so it transforms f in place.

Arguments

Type	Intent		Name
class(cuda_poisson_000_t)		::	self
class(field_t),	intent(inout)	::	f
class(field_t),	intent(inout)	::	temp

public subroutine fft_forward_110_cuda(self, f)

Forward FFT for 110 case: transpose (nx,ny,nz)->(nz,nx,ny) then R2C

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_110_t)				::	self
class(field_t),	intent(in)			::	f

public subroutine fft_backward_110_cuda(self, f)

Backward FFT for 110 case: C2R then transpose (nz,nx,ny)->(nx,ny,nz)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_110_t)				::	self
class(field_t),	intent(inout)			::	f

public subroutine redistribute_110_to_yslab(self, zslab, yslab)

Redistributes the 110 field from the physical Z-slab (nx, ny, nz_l) to the cuFFTMp Y-slab real view (2*(nz/2+1), nx, ny_l), ahead of the forward transform. See the comment above pack_110_xyz_to_yslab.

Arguments

Type	Intent	Attributes		Name
class(cuda_poisson_110_t)			::	self
real(kind=dp),	intent(in),	device, dimension(:, :, :)	::	zslab
real(kind=dp),	intent(inout),	device, dimension(:, :, :)	::	yslab

public subroutine redistribute_110_to_zslab(self, yslab, zslab)

Inverse of redistribute_110_to_yslab, after the backward transform.

Arguments

Type	Intent	Attributes		Name
class(cuda_poisson_110_t)			::	self
real(kind=dp),	intent(in),	device, dimension(:, :, :)	::	yslab
real(kind=dp),	intent(inout),	device, dimension(:, :, :)	::	zslab

public subroutine fft_forward_100_cuda(self, f)

Forward FFT for non-periodic-X case We transpose X<->Y so that the non-periodic direction becomes the "Y" direction in the transposed space, then use the same FFT approach as 010

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_100_t)				::	self
class(field_t),	intent(in)			::	f

public subroutine fft_backward_100_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_100_t)				::	self
class(field_t),	intent(inout)			::	f

public subroutine fft_forward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

public subroutine fft_backward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

public subroutine fft_postprocess_000_cuda(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_000_t)				::	self

public subroutine fft_postprocess_100_cuda(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_100_t)				::	self

public subroutine exchange_dim2_mirror(self, c_dev)

Fills c_mirror_dev and c_plane_recv_dev with the dim2 mirror of the local spectral slab, so that the paired split can read its partner mode without going looking for it in another rank's memory.

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
complex(kind=dp),	intent(in),		device, dimension(:, :, :)	::	c_dev

public subroutine fft_postprocess_010_cuda(self)

if stretching in y is 'bottom'

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_010_t)				::	self

public subroutine fft_postprocess_110_cuda(self)

Spectral post-processing for 110 case (R2C Z-transpose). Spectral array is (nz/2+1, nx, ny) = (nx_spec, ny_spec, nz_spec). Thread i->X (ny_spec), blockIdx%y k->Y (nz_spec), serial j->Z (nx_spec). On a single rank X and Y are both whole, so 7 kernel launches avoid cross-block race conditions. On multiple ranks cuFFTMp splits X across ranks, so the X pairing reaches its partner through the dim2 mirror exchange (see exchange_dim2_mirror) instead, adding two more launches.

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_110_t)				::	self

public subroutine enforce_periodicity_x_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_100_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine undo_periodicity_x_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_100_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine enforce_periodicity_y_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_010_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine undo_periodicity_y_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_010_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine enforce_periodicity_xy_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_110_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

public subroutine undo_periodicity_xy_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_110_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

m_cuda_poisson_fft Module

Contents

Interfaces

Derived Types

Functions

Subroutines

Uses

Used by

Interfaces

interface

public function cufftExecR2C_C(plan, idata, odata) bind(C, name='cufftExecR2C')

Arguments

Return Value integer(kind=c_int)

interface

public function cufftExecC2R_C(plan, idata, odata) bind(C, name='cufftExecC2R')

Arguments

Return Value integer(kind=c_int)

Derived Types

type, public, abstract, extends(poisson_fft_t) :: cuda_poisson_fft_t

Components

Type-Bound Procedures

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_000_t

Components

Type-Bound Procedures

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_010_t

Components

Type-Bound Procedures

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_100_t

Components

Type-Bound Procedures

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_110_t

Components

Type-Bound Procedures

Functions

public pure function cuda_fft_decomp_reason(bc_code, cdims, nproc_dir) result(reason)

Arguments

Return Value character(len=110)

Subroutines

public subroutine create_fft_plan(plan, use_cufftmp, nx, ny, nz, plan_type, plan_name, worksize_out, stat, auto_alloc_off)

Arguments

public subroutine create_fft_plan_pair(plan_fw, plan_bw, use_cufftmp, fft_n, fw_type, bw_type, label, worksize_fw, worksize_bw, stat, no_auto_alloc)

Arguments

public pure subroutine cuda_fft_spectral_dims(bc_code, dims_glob, dims_loc, nproc_dir, nrank_dir, n_spec, n_sp_st)

Arguments

public pure subroutine cuda_fft_plan_setup(bc_code, dims_glob, dims_loc, fft_n, fft_n_loc, fw_type, bw_type)

Arguments

public subroutine create_cuda_poisson_fft(poisson_fft, mesh, xdirps, ydirps, zdirps, lowmem)

Arguments

public subroutine init_case(self, mesh)

Arguments

public subroutine init_case_010(self, mesh)

Arguments

public subroutine init_case_100(self, mesh)

Arguments

public subroutine init_case_110(self, mesh)

Arguments

public subroutine init_dim2_mirror(self, mesh)

Arguments

public subroutine create_plans(self, mesh, fft_n, fft_n_loc, fw_plan_type, bw_plan_type)

Arguments

public subroutine create_plans_110(self, mesh, fft_n, fft_n_loc, fw_plan_type, bw_plan_type)

Arguments

public subroutine solve_poisson_000_cuda(self, f, temp)

Arguments

public subroutine fft_forward_110_cuda(self, f)

Arguments

public subroutine fft_backward_110_cuda(self, f)

Arguments

public subroutine redistribute_110_to_yslab(self, zslab, yslab)

Arguments

public subroutine redistribute_110_to_zslab(self, yslab, zslab)

Arguments

public subroutine fft_forward_100_cuda(self, f)

Arguments

public subroutine fft_backward_100_cuda(self, f)

Arguments

public subroutine fft_forward_cuda(self, f)

Arguments

public subroutine fft_backward_cuda(self, f)

Arguments