cuda_poisson_110_t Derived Type

type, public, extends(cuda_poisson_fft_t) :: cuda_poisson_110_t

Non-periodic in x and y, periodic in z


Inherits

type~~cuda_poisson_110_t~~InheritsGraph type~cuda_poisson_110_t cuda_poisson_110_t type~cuda_poisson_fft_t cuda_poisson_fft_t type~cuda_poisson_110_t->type~cuda_poisson_fft_t cudaLibXtDesc cudaLibXtDesc type~cuda_poisson_fft_t->cudaLibXtDesc xtdesc type~poisson_fft_t poisson_fft_t type~cuda_poisson_fft_t->type~poisson_fft_t type~mesh_t mesh_t type~poisson_fft_t->type~mesh_t mesh type~geo_t geo_t type~mesh_t->type~geo_t geo type~grid_t grid_t type~mesh_t->type~grid_t grid type~par_t par_t type~mesh_t->type~par_t par

Components

Type Visibility Attributes Name Initial
type(mesh_t), public, pointer :: mesh => null()
integer, public :: nx_glob

Global dimensions

integer, public :: ny_glob

Global dimensions

integer, public :: nz_glob

Global dimensions

integer, public :: nx_loc

Local dimensions

integer, public :: ny_loc

Local dimensions

integer, public :: nz_loc

Local dimensions

integer, public :: nx_perm

Local dimensions in the permuted slabs

integer, public :: ny_perm

Local dimensions in the permuted slabs

integer, public :: nz_perm

Local dimensions in the permuted slabs

integer, public :: nx_spec

Local dimensions in the permuted slabs in spectral space

integer, public :: ny_spec

Local dimensions in the permuted slabs in spectral space

integer, public :: nz_spec

Local dimensions in the permuted slabs in spectral space

integer, public :: sp_st(3)

Offset per spectral dimension (dim1, dim2, dim3) in spectral space

complex(kind=dp), public, allocatable, dimension(:, :, :) :: waves

Local domain sized array storing the spectral equivalence constants

real(kind=dp), public, allocatable, dimension(:) :: ax

Wave numbers in x, y, and z

real(kind=dp), public, allocatable, dimension(:) :: bx

Wave numbers in x, y, and z

real(kind=dp), public, allocatable, dimension(:) :: ay

Wave numbers in x, y, and z

real(kind=dp), public, allocatable, dimension(:) :: by

Wave numbers in x, y, and z

real(kind=dp), public, allocatable, dimension(:) :: az

Wave numbers in x, y, and z

real(kind=dp), public, allocatable, dimension(:) :: bz

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: kx

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: ky

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: kz

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: exs

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: eys

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: ezs

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: k2x

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: k2y

Wave numbers in x, y, and z

complex(kind=dp), public, allocatable, dimension(:) :: k2z

Wave numbers in x, y, and z

real(kind=dp), public, allocatable, dimension(:) :: trans_x_re

Staggered grid transformation

real(kind=dp), public, allocatable, dimension(:) :: trans_x_im

Staggered grid transformation

real(kind=dp), public, allocatable, dimension(:) :: trans_y_re

Staggered grid transformation

real(kind=dp), public, allocatable, dimension(:) :: trans_y_im

Staggered grid transformation

real(kind=dp), public, allocatable, dimension(:) :: trans_z_re

Staggered grid transformation

real(kind=dp), public, allocatable, dimension(:) :: trans_z_im

Staggered grid transformation

logical, public :: periodic_x

Periodicity in x, y, and z

logical, public :: periodic_y

Periodicity in x, y, and z

logical, public :: periodic_z

Periodicity in x, y, and z

logical, public :: stretched_y = .false.

Periodicity in x, y, and z

logical, public :: stretched_y_sym

Periodicity in x, y, and z

character(len=3), public :: bc_code

Boundary condition case, see poisson_bc_code. The backend's init sets it first, ahead of base_init.

real(kind=dp), public, allocatable, dimension(:, :, :, :) :: a_odd_re

Stretching operator matrices

real(kind=dp), public, allocatable, dimension(:, :, :, :) :: a_odd_im

Stretching operator matrices

real(kind=dp), public, allocatable, dimension(:, :, :, :) :: a_even_re

Stretching operator matrices

real(kind=dp), public, allocatable, dimension(:, :, :, :) :: a_even_im

Stretching operator matrices

real(kind=dp), public, allocatable, dimension(:, :, :, :) :: a_re

Stretching operator matrices

real(kind=dp), public, allocatable, dimension(:, :, :, :) :: a_im

Stretching operator matrices

logical, public :: lowmem = .false.

lowmem option, only used in CUDA backend

complex(kind=dp), public, device, allocatable, dimension(:, :, :) :: waves_dev

Local domain sized array storing the spectral equivalence constants

real(kind=dp), public, device, allocatable, dimension(:) :: ax_dev

Wave numbers in x, y, and z

real(kind=dp), public, device, allocatable, dimension(:) :: bx_dev

Wave numbers in x, y, and z

real(kind=dp), public, device, allocatable, dimension(:) :: ay_dev

Wave numbers in x, y, and z

real(kind=dp), public, device, allocatable, dimension(:) :: by_dev

Wave numbers in x, y, and z

real(kind=dp), public, device, allocatable, dimension(:) :: az_dev

Wave numbers in x, y, and z

real(kind=dp), public, device, allocatable, dimension(:) :: bz_dev

Wave numbers in x, y, and z

integer, public :: plan3D_fw

Forward and backward FFT transform plans

integer, public :: plan3D_bw

Forward and backward FFT transform plans

logical, public :: use_cufftmp = .true.

Flag to indicate whether cuFFTMp is used

type(cudaLibXtDesc), public, pointer :: xtdesc

cuFFTMp object manages decomposition and data storage

complex(kind=dp), public, device, allocatable, dimension(:, :, :) :: c_dev

Standard cuFFT storage

real(kind=dp), public, device, allocatable, dimension(:, :, :) :: r_dev

cuFFT real workspace (input/output of R2C/C2R)

complex(kind=dp), public, device, allocatable, dimension(:, :, :) :: c_mirror_dev

dim2 mirror of the local spectral slab, used by the 010, 100 and 110 cases to reach the paired split partner that lives on another rank. Only init_dim2_mirror allocates it, and init_cuda reads its allocation status as "this case needs cuFFTMp".

complex(kind=dp), public, device, allocatable, dimension(:, :, :) :: c_slab_send_dev

Staging copy of the local slab, sent in place of c_dev itself. In the cuFFTMp path c_dev is a c_f_pointer onto descriptor%data(1), which is cufftXtMalloc memory and therefore lives in the NVSHMEM symmetric heap. UCX does not reliably classify such a pointer as device memory: when it falls back to the host CMA transport it calls process_vm_readv on it and the job aborts with EFAULT. Copying into ordinary device memory first makes the exchange independent of which transport UCX happens to select.

complex(kind=dp), public, device, allocatable, dimension(:, :) :: c_plane_send_dev

The single dim2 plane that falls outside the mirrored slab

complex(kind=dp), public, device, allocatable, dimension(:, :) :: c_plane_recv_dev

The single dim2 plane that falls outside the mirrored slab

integer, public :: mirror_slab_rank = 0

Ranks holding the mirror slab and the extra mirror plane

integer, public :: mirror_plane_rank = 0

Ranks holding the mirror slab and the extra mirror plane

real(kind=dp), public, device, allocatable, dimension(:, :, :) :: r_dev_110

Transposed real workspace (nz, nx, ny), single rank only

real(kind=dp), public, device, allocatable, dimension(:, :, :, :) :: r_a2a_send_dev

All-to-all buffers redistributing the field between the physical Z-slab (nx, ny, nz_l) and the cuFFTMp Y-slab (2*(nz/2+1), nx, ny_l), sized (nz_l, nx, ny_l, nproc). Ordinary device memory, for the same reason as c_slab_send_dev.

real(kind=dp), public, device, allocatable, dimension(:, :, :, :) :: r_a2a_recv_dev

All-to-all buffers redistributing the field between the physical Z-slab (nx, ny, nz_l) and the cuFFTMp Y-slab (2*(nz/2+1), nx, ny_l), sized (nz_l, nx, ny_l, nproc). Ordinary device memory, for the same reason as c_slab_send_dev.


Type-Bound Procedures

procedure, public :: base_init

  • public subroutine base_init(self, mesh, xdirps, ydirps, zdirps, n_spec, n_sp_st)

    Sets up the state every case shares. The backend's init calls it once it has set bc_code, which the spectral extents it passes in already depend on. The supported BC codes are the cases of the backend factories, create_omp_poisson_fft and create_cuda_poisson_fft.

    Arguments

    Type IntentOptional Attributes Name
    class(poisson_fft_t) :: self
    type(mesh_t), intent(in), target :: mesh
    type(dirps_t), intent(in) :: xdirps
    type(dirps_t), intent(in) :: ydirps
    type(dirps_t), intent(in) :: zdirps
    integer, intent(in), dimension(3) :: n_spec
    integer, intent(in), dimension(3) :: n_sp_st

procedure, public :: solve_poisson

  • public subroutine solve_poisson(self, f, temp)

    Gathers each non-periodic line into a periodic one, so that a plain FFT can stand in for the cosine transform, solves in spectral space, and scatters the result back. The 000 case has nothing to reorder and overrides this to transform f in place.

    Arguments

    Type IntentOptional Attributes Name
    class(poisson_fft_t) :: self
    class(field_t), intent(inout) :: f
    class(field_t), intent(inout) :: temp

procedure, public :: stretching_matrix

  • public subroutine stretching_matrix(self, geo, xdirps, ydirps, zdirps)

    Stretching necessitates a special operation in spectral space. The coefficients for the operation are stored in matrix form.

    Read more…

    Arguments

    Type IntentOptional Attributes Name
    class(poisson_fft_t) :: self
    type(geo_t), intent(in) :: geo
    type(dirps_t), intent(in) :: xdirps
    type(dirps_t), intent(in) :: ydirps
    type(dirps_t), intent(in) :: zdirps

procedure, public :: waves_set

  • public subroutine waves_set(self, geo, xdirps, ydirps, zdirps)

    Spectral equivalence constants

    Read more…

    Arguments

    Type IntentOptional Attributes Name
    class(poisson_fft_t) :: self
    type(geo_t), intent(in) :: geo
    type(dirps_t), intent(in) :: xdirps
    type(dirps_t), intent(in) :: ydirps
    type(dirps_t), intent(in) :: zdirps

procedure, public :: get_km

  • public function get_km(self, i, j, k) result(km)

    Arguments

    Type IntentOptional Attributes Name
    class(poisson_fft_t) :: self
    integer, intent(in) :: i
    integer, intent(in) :: j
    integer, intent(in) :: k

    Return Value complex(kind=dp)

procedure, public :: get_km_re

  • public function get_km_re(self, i, j, k) result(re)

    Arguments

    Type IntentOptional Attributes Name
    class(poisson_fft_t) :: self
    integer, intent(in) :: i
    integer, intent(in) :: j
    integer, intent(in) :: k

    Return Value real(kind=dp)

procedure, public :: get_km_im

  • public function get_km_im(self, i, j, k) result(re)

    Arguments

    Type IntentOptional Attributes Name
    class(poisson_fft_t) :: self
    integer, intent(in) :: i
    integer, intent(in) :: j
    integer, intent(in) :: k

    Return Value real(kind=dp)

procedure, public :: init => init_cuda

  • private subroutine init_cuda(self, mesh, xdirps, ydirps, zdirps, lowmem)

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_fft_t) :: self
    type(mesh_t), intent(in), target :: mesh
    type(dirps_t), intent(in) :: xdirps
    type(dirps_t), intent(in) :: ydirps
    type(dirps_t), intent(in) :: zdirps
    logical, intent(in), optional :: lowmem

procedure, public :: init_dim2_mirror

  • public subroutine init_dim2_mirror(self, mesh)

    The 010, 100 and 110 cases pair each dim2 spectral mode with its mirror, which cuFFTMp may place on another rank. Slab index r of P owns global dim2 modes [rm + 1, (r + 1)m] with m = ny_spec, and the mirror of that range is slab P - 1 - r shifted up by one plane, so the extra plane comes from slab P - r. Both partners are symmetric. A single rank holds every partner itself and allocates nothing. This is the only place c_mirror_dev is allocated, see the cuFFTMp check in init_cuda.

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_fft_t) :: self
    type(mesh_t), intent(in) :: mesh

procedure, public :: exchange_dim2_mirror

  • public subroutine exchange_dim2_mirror(self, c_dev)

    Fills c_mirror_dev and c_plane_recv_dev with the dim2 mirror of the local spectral slab, so that the paired split can read its partner mode without going looking for it in another rank's memory.

    Read more…

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_fft_t) :: self
    complex(kind=dp), intent(in), device, dimension(:, :, :) :: c_dev

procedure, public :: init_case => init_case_110

procedure, public :: create_plans => create_plans_110

  • public subroutine create_plans_110(self, mesh, fft_n, fft_n_loc, fw_plan_type, bw_plan_type)

    R2C with Z-transpose, on (nz, nx, ny). A single rank keeps the plain cuFFT path; on multiple ranks cuFFTMp distributes the transform, and the physical Z-slab is redistributed into its Y-slab input through an MPI_Alltoall (see redistribute_110_to_yslab).

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_110_t) :: self
    type(mesh_t), intent(in) :: mesh
    integer, intent(in) :: fft_n(3)
    integer, intent(in) :: fft_n_loc(2)
    integer, intent(in) :: fw_plan_type
    integer, intent(in) :: bw_plan_type

procedure, public :: fft_forward => fft_forward_110_cuda

  • public subroutine fft_forward_110_cuda(self, f)

    Forward FFT for 110 case: transpose (nx,ny,nz)->(nz,nx,ny) then R2C

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_110_t) :: self
    class(field_t), intent(in) :: f

procedure, public :: fft_backward => fft_backward_110_cuda

  • public subroutine fft_backward_110_cuda(self, f)

    Backward FFT for 110 case: C2R then transpose (nz,nx,ny)->(nx,ny,nz)

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_110_t) :: self
    class(field_t), intent(inout) :: f

procedure, public :: fft_postprocess => fft_postprocess_110_cuda

  • public subroutine fft_postprocess_110_cuda(self)

    Spectral post-processing for 110 case (R2C Z-transpose). Spectral array is (nz/2+1, nx, ny) = (nx_spec, ny_spec, nz_spec). Thread i->X (ny_spec), blockIdx%y k->Y (nz_spec), serial j->Z (nx_spec). On a single rank X and Y are both whole, so 7 kernel launches avoid cross-block race conditions. On multiple ranks cuFFTMp splits X across ranks, so the X pairing reaches its partner through the dim2 mirror exchange (see exchange_dim2_mirror) instead, adding two more launches.

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_110_t) :: self

procedure, public :: enforce_periodicity => enforce_periodicity_xy_cuda

procedure, public :: undo_periodicity => undo_periodicity_xy_cuda

procedure, public :: redistribute_110_to_yslab

  • public subroutine redistribute_110_to_yslab(self, zslab, yslab)

    Redistributes the 110 field from the physical Z-slab (nx, ny, nz_l) to the cuFFTMp Y-slab real view (2*(nz/2+1), nx, ny_l), ahead of the forward transform. See the comment above pack_110_xyz_to_yslab.

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_110_t) :: self
    real(kind=dp), intent(in), device, dimension(:, :, :) :: zslab
    real(kind=dp), intent(inout), device, dimension(:, :, :) :: yslab

procedure, public :: redistribute_110_to_zslab

  • public subroutine redistribute_110_to_zslab(self, yslab, zslab)

    Inverse of redistribute_110_to_yslab, after the backward transform.

    Arguments

    Type IntentOptional Attributes Name
    class(cuda_poisson_110_t) :: self
    real(kind=dp), intent(in), device, dimension(:, :, :) :: yslab
    real(kind=dp), intent(inout), device, dimension(:, :, :) :: zslab