cuda_poisson_fft_t

type, public, extends(poisson_fft_t) :: cuda_poisson_fft_t

FFT based Poisson solver

Inherits

Help

Components

Type	Visibility	Attributes		Name		Initial
type(mesh_t),	public,	pointer	::	mesh	=>	null()
integer,	public		::	nx_glob			Global dimensions
integer,	public		::	ny_glob			Global dimensions
integer,	public		::	nz_glob			Global dimensions
integer,	public		::	nx_loc			Local dimensions
integer,	public		::	ny_loc			Local dimensions
integer,	public		::	nz_loc			Local dimensions
integer,	public		::	nx_perm			Local dimensions in the permuted slabs
integer,	public		::	ny_perm			Local dimensions in the permuted slabs
integer,	public		::	nz_perm			Local dimensions in the permuted slabs
integer,	public		::	nx_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	ny_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	nz_spec			Local dimensions in the permuted slabs in spectral space
integer,	public		::	sp_st(3)			Offset per spectral dimension (dim1, dim2, dim3) in spectral space
complex(kind=dp),	public,	allocatable, dimension(:, :, :)	::	waves			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	allocatable, dimension(:)	::	ax			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bx			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	ay			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	by			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	az			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	bz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kx			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ky			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	kz			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	exs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	eys			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	ezs			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2x			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2y			Wave numbers in x, y, and z
complex(kind=dp),	public,	allocatable, dimension(:)	::	k2z			Wave numbers in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_re			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_x_im			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_re			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_y_im			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_re			Staggared grid transformation
real(kind=dp),	public,	allocatable, dimension(:)	::	trans_z_im			Staggared grid transformation
logical,	public		::	periodic_x			Periodicity in x, y, and z
logical,	public		::	periodic_y			Periodicity in x, y, and z
logical,	public		::	periodic_z			Periodicity in x, y, and z
logical,	public		::	stretched_y	=	.false.	Periodicity in x, y, and z
logical,	public		::	stretched_y_sym			Periodicity in x, y, and z
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_odd_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_even_im			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_re			Stretching operator matrices
real(kind=dp),	public,	allocatable, dimension(:, :, :, :)	::	a_im			Stretching operator matrices
logical,	public		::	lowmem	=	.false.	lowmem option, only used in CUDA backend
procedure(poisson_xxx),	public,	pointer	::	poisson	=>	null()	Procedure pointer to BC specific poisson solvers
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	waves_dev			Local domain sized array storing the spectral equivalence constants
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ax_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bx_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	ay_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	by_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	az_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:)	::	bz_dev			Wave numbers in x, y, and z
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_odd_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_odd_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_even_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_even_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_re_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	store_a_im_dev			Stretching operator matrices stores
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_odd_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_odd_im_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_even_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_even_im_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_re_dev			Stretching operator matrices
real(kind=dp),	public,	device, allocatable, dimension(:, :, :, :)	::	a_im_dev			Stretching operator matrices
integer,	public		::	plan3D_fw			Forward and backward FFT transform plans
integer,	public		::	plan3D_bw			Forward and backward FFT transform plans
logical,	public		::	use_cufftmp	=	.true.	Flag to indicate whether cuFFTMp is used
logical,	public		::	is_000_case	=	.false.	Flag for cases
logical,	public		::	is_010_case	=	.false.
logical,	public		::	is_100_case	=	.false.
logical,	public		::	is_110_case	=	.false.
type(cudaLibXtDesc),	public,	pointer	::	xtdesc			cuFFTMp object manages decomposition and data storage
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_dev			Standard cuFFT storage
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev			cuFFT real workspace (input/output of R2C/C2R)
real(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	r_dev_110			Transposed real workspace for 110 case (nz, nx, ny)
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_mirror_dev			dim2 mirror of the local spectral slab, used by the 100 case to reach the paired split partner that lives on another rank
complex(kind=dp),	public,	device, allocatable, dimension(:, :, :)	::	c_slab_send_dev			Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_send_dev			The single dim2 plane that falls outside the mirrored slab
complex(kind=dp),	public,	device, allocatable, dimension(:, :)	::	c_plane_recv_dev			The single dim2 plane that falls outside the mirrored slab
integer,	public		::	mirror_slab_rank	=	0	Ranks holding the mirror slab and the extra mirror plane
integer,	public		::	mirror_plane_rank	=	0	Ranks holding the mirror slab and the extra mirror plane

Constructor

public interface cuda_poisson_fft_t

private function init(mesh, xdirps, ydirps, zdirps, lowmem) result(poisson_fft)

if stretching in y is 'bottom'

Arguments

Type	Intent	Optional	Attributes		Name
type(mesh_t),	intent(in),		target	::	mesh
type(dirps_t),	intent(in)			::	xdirps
type(dirps_t),	intent(in)			::	ydirps
type(dirps_t),	intent(in)			::	zdirps
logical,	intent(in),	optional		::	lowmem

Return Value type(cuda_poisson_fft_t)

Type-Bound Procedures

procedure, public :: base_init

public subroutine base_init(self, mesh, xdirps, ydirps, zdirps, n_spec, n_sp_st)

Arguments

Type	Intent	Attributes		Name
class(poisson_fft_t)			::	self
type(mesh_t),	intent(in),	target	::	mesh
type(dirps_t),	intent(in)		::	xdirps
type(dirps_t),	intent(in)		::	ydirps
type(dirps_t),	intent(in)		::	zdirps
integer,	intent(in),	dimension(3)	::	n_spec
integer,	intent(in),	dimension(3)	::	n_sp_st

procedure, public :: solve_poisson

public subroutine solve_poisson(self, f, temp)

Arguments

Type	Intent		Name
class(poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f
class(field_t),	intent(inout)	::	temp

procedure, public :: stretching_matrix

public subroutine stretching_matrix(self, geo, xdirps, ydirps, zdirps)

Stretching necessitates a special operation in spectral space. The coefficients for the operation are stored in matrix form.

Arguments

Type	Intent		Name
class(poisson_fft_t)		::	self
type(geo_t),	intent(in)	::	geo
type(dirps_t),	intent(in)	::	xdirps
type(dirps_t),	intent(in)	::	ydirps
type(dirps_t),	intent(in)	::	zdirps

procedure, public :: waves_set

public subroutine waves_set(self, geo, xdirps, ydirps, zdirps)

Spectral equivalence constants

Arguments

Type	Intent		Name
class(poisson_fft_t)		::	self
type(geo_t),	intent(in)	::	geo
type(dirps_t),	intent(in)	::	xdirps
type(dirps_t),	intent(in)	::	ydirps
type(dirps_t),	intent(in)	::	zdirps

procedure, public :: get_km

public function get_km(self, i, j, k) result(km)

Arguments

Type	Intent		Name
class(poisson_fft_t)		::	self
integer,	intent(in)	::	i
integer,	intent(in)	::	j
integer,	intent(in)	::	k

Return Value complex(kind=dp)

procedure, public :: get_km_re

public function get_km_re(self, i, j, k) result(re)

Arguments

Type	Intent		Name
class(poisson_fft_t)		::	self
integer,	intent(in)	::	i
integer,	intent(in)	::	j
integer,	intent(in)	::	k

Return Value real(kind=dp)

procedure, public :: get_km_im

public function get_km_im(self, i, j, k) result(re)

Arguments

Type	Intent		Name
class(poisson_fft_t)		::	self
integer,	intent(in)	::	i
integer,	intent(in)	::	j
integer,	intent(in)	::	k

Return Value real(kind=dp)

procedure, public :: fft_forward => fft_forward_cuda

public subroutine fft_forward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

procedure, public :: fft_forward_010 => fft_forward_cuda

public subroutine fft_forward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

procedure, public :: fft_forward_100 => fft_forward_100_cuda

public subroutine fft_forward_100_cuda(self, f)

Forward FFT for non-periodic-X case We transpose X<->Y so that the non-periodic direction becomes the "Y" direction in the transposed space, then use the same FFT approach as 010

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

procedure, public :: fft_forward_110 => fft_forward_110_cuda

public subroutine fft_forward_110_cuda(self, f)

Forward FFT for 110 case: transpose (nx,ny,nz)->(nz,nx,ny) then R2C

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(in)			::	f

procedure, public :: fft_backward => fft_backward_cuda

public subroutine fft_backward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

procedure, public :: fft_backward_010 => fft_backward_cuda

public subroutine fft_backward_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

procedure, public :: fft_backward_100 => fft_backward_100_cuda

public subroutine fft_backward_100_cuda(self, f)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

procedure, public :: fft_backward_110 => fft_backward_110_cuda

public subroutine fft_backward_110_cuda(self, f)

Backward FFT for 110 case: C2R then transpose (nz,nx,ny)->(nx,ny,nz)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
class(field_t),	intent(inout)			::	f

procedure, public :: fft_postprocess_000 => fft_postprocess_000_cuda

public subroutine fft_postprocess_000_cuda(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

procedure, public :: fft_postprocess_010 => fft_postprocess_010_cuda

public subroutine fft_postprocess_010_cuda(self)

if stretching in y is 'bottom'

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

procedure, public :: fft_postprocess_100 => fft_postprocess_100_cuda

public subroutine fft_postprocess_100_cuda(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

procedure, public :: fft_postprocess_110 => fft_postprocess_110_cuda

public subroutine fft_postprocess_110_cuda(self)

Spectral post-processing for 110 case (R2C Z-transpose) 7 separate kernel launches to avoid cross-block race conditions. Spectral array is (nz/2+1, nx, ny) = (nx_spec, ny_spec, nz_spec). Thread i->X (ny_spec), blockIdx%y k->Y (nz_spec), serial j->Z (nx_spec).

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self

procedure, public :: enforce_periodicity_x => enforce_periodicity_x_cuda

public subroutine enforce_periodicity_x_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

procedure, public :: undo_periodicity_x => undo_periodicity_x_cuda

public subroutine undo_periodicity_x_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

procedure, public :: enforce_periodicity_y => enforce_periodicity_y_cuda

public subroutine enforce_periodicity_y_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

procedure, public :: undo_periodicity_y => undo_periodicity_y_cuda

public subroutine undo_periodicity_y_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

procedure, public :: enforce_periodicity_xy => enforce_periodicity_xy_cuda

public subroutine enforce_periodicity_xy_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

procedure, public :: undo_periodicity_xy => undo_periodicity_xy_cuda

public subroutine undo_periodicity_xy_cuda(self, f_out, f_in)

Arguments

Type	Intent		Name
class(cuda_poisson_fft_t)		::	self
class(field_t),	intent(inout)	::	f_out
class(field_t),	intent(in)	::	f_in

procedure, public :: exchange_mirror_100

public subroutine exchange_mirror_100(self, c_dev)

Fills c_mirror_dev and c_plane_recv_dev with the dim2 mirror of the local spectral slab, so that the 100 paired split can read its partner mode without going looking for it in another rank's memory.

Arguments

Type	Intent	Optional	Attributes		Name
class(cuda_poisson_fft_t)				::	self
complex(kind=dp),	intent(in),		device, dimension(:, :, :)	::	c_dev

cuda_poisson_fft_t Derived Type

Contents

Variables

Constructor

Type-Bound Procedures

type, public, extends(poisson_fft_t) :: cuda_poisson_fft_t

Inherits

Components

Constructor

public interface cuda_poisson_fft_t

private function init(mesh, xdirps, ydirps, zdirps, lowmem) result(poisson_fft)

Arguments

Return Value type(cuda_poisson_fft_t)

Type-Bound Procedures

procedure, public :: base_init

public subroutine base_init(self, mesh, xdirps, ydirps, zdirps, n_spec, n_sp_st)

Arguments

procedure, public :: solve_poisson

public subroutine solve_poisson(self, f, temp)

Arguments

procedure, public :: stretching_matrix

public subroutine stretching_matrix(self, geo, xdirps, ydirps, zdirps)

Arguments

procedure, public :: waves_set

public subroutine waves_set(self, geo, xdirps, ydirps, zdirps)

Arguments

procedure, public :: get_km

public function get_km(self, i, j, k) result(km)

Arguments

Return Value complex(kind=dp)

procedure, public :: get_km_re

public function get_km_re(self, i, j, k) result(re)

Arguments

Return Value real(kind=dp)

procedure, public :: get_km_im

public function get_km_im(self, i, j, k) result(re)

Arguments

Return Value real(kind=dp)

procedure, public :: fft_forward => fft_forward_cuda

public subroutine fft_forward_cuda(self, f)

Arguments

procedure, public :: fft_forward_010 => fft_forward_cuda

public subroutine fft_forward_cuda(self, f)

Arguments

procedure, public :: fft_forward_100 => fft_forward_100_cuda

public subroutine fft_forward_100_cuda(self, f)

Arguments

procedure, public :: fft_forward_110 => fft_forward_110_cuda

public subroutine fft_forward_110_cuda(self, f)

Arguments

procedure, public :: fft_backward => fft_backward_cuda

public subroutine fft_backward_cuda(self, f)

Arguments

procedure, public :: fft_backward_010 => fft_backward_cuda

public subroutine fft_backward_cuda(self, f)

Arguments

procedure, public :: fft_backward_100 => fft_backward_100_cuda

public subroutine fft_backward_100_cuda(self, f)

Arguments

procedure, public :: fft_backward_110 => fft_backward_110_cuda

public subroutine fft_backward_110_cuda(self, f)

Arguments

procedure, public :: fft_postprocess_000 => fft_postprocess_000_cuda

public subroutine fft_postprocess_000_cuda(self)

Arguments

procedure, public :: fft_postprocess_010 => fft_postprocess_010_cuda

public subroutine fft_postprocess_010_cuda(self)

Arguments

procedure, public :: fft_postprocess_100 => fft_postprocess_100_cuda

public subroutine fft_postprocess_100_cuda(self)

Arguments

procedure, public :: fft_postprocess_110 => fft_postprocess_110_cuda

public subroutine fft_postprocess_110_cuda(self)

Arguments

procedure, public :: enforce_periodicity_x => enforce_periodicity_x_cuda

public subroutine enforce_periodicity_x_cuda(self, f_out, f_in)

Arguments

procedure, public :: undo_periodicity_x => undo_periodicity_x_cuda

public subroutine undo_periodicity_x_cuda(self, f_out, f_in)

Arguments