spectral_processing.f90 Source File


This file depends on

sourcefile~~spectral_processing.f90~2~~EfferentGraph sourcefile~spectral_processing.f90~2 spectral_processing.f90 sourcefile~common.f90~3 common.f90 sourcefile~spectral_processing.f90~2->sourcefile~common.f90~3 sourcefile~logger.f90 logger.f90 sourcefile~common.f90~3->sourcefile~logger.f90

Files dependent on this one

sourcefile~~spectral_processing.f90~2~~AfferentGraph sourcefile~spectral_processing.f90~2 spectral_processing.f90 sourcefile~poisson_fft.f90 poisson_fft.f90 sourcefile~poisson_fft.f90->sourcefile~spectral_processing.f90~2 sourcefile~backend.f90~4 backend.f90 sourcefile~backend.f90~4->sourcefile~poisson_fft.f90 sourcefile~estimate.f90 estimate.f90 sourcefile~estimate.f90->sourcefile~poisson_fft.f90 sourcefile~context.f90 context.f90 sourcefile~estimate.f90->sourcefile~context.f90 sourcefile~memcheck_device.f90 memcheck_device.f90 sourcefile~memcheck_device.f90->sourcefile~poisson_fft.f90 sourcefile~build.f90 build.f90 sourcefile~build.f90->sourcefile~estimate.f90 sourcefile~build.f90->sourcefile~context.f90 sourcefile~report.f90 report.f90 sourcefile~build.f90->sourcefile~report.f90 sourcefile~setup.f90 setup.f90 sourcefile~build.f90->sourcefile~setup.f90 sourcefile~context.f90->sourcefile~memcheck_device.f90 sourcefile~memcheck.f90 memcheck.f90 sourcefile~memcheck.f90->sourcefile~estimate.f90 sourcefile~memcheck.f90->sourcefile~build.f90 sourcefile~memcheck.f90->sourcefile~context.f90 sourcefile~memcheck.f90->sourcefile~report.f90 sourcefile~report.f90->sourcefile~estimate.f90 sourcefile~report.f90->sourcefile~memcheck_device.f90 sourcefile~report.f90->sourcefile~context.f90 sourcefile~setup.f90->sourcefile~backend.f90~4 sourcefile~poisson_solver.f90 poisson_solver.f90 sourcefile~poisson_solver.f90->sourcefile~setup.f90 sourcefile~xcompact.f90 xcompact.f90 sourcefile~xcompact.f90->sourcefile~setup.f90

Source Code

module m_cuda_spectral
  use cudafor

  use m_common, only: dp

  implicit none

  ! Threshold below which a spectral coefficient is treated as zero. Must
  ! scale with the working precision: single-precision roundoff leaves
  ! ~1e-13 noise in wave numbers that should vanish (e.g. Nyquist modes),
  ! and dividing by that noise corrupts the pressure field.
  real(dp), parameter :: eps_wave = epsilon(1._dp)

contains

  ! ------------------------------------------------------------------
  ! SHARED POINTWISE TRANSFORMS
  !
  ! The 010 and 100 cases run the same algebra on transposed data, so
  ! these hold the single copy of it. Named by dimension rather than by
  ! physical direction: dim1 is the R2C direction, dim2 is the paired
  ! non-periodic one, dim3 is the remaining periodic one.
  ! ------------------------------------------------------------------

  attributes(device) subroutine postprocess_dim13_fw(div_r, div_c, i1, i3, &
                                                     n1, n3, a1, b1, a3, b3)
    !! Forward postprocess of one spectral entry in dim3 then dim1.
    implicit none

    real(dp), intent(inout) :: div_r, div_c
    integer, intent(in) :: i1, i3, n1, n3
    real(dp), intent(in) :: a1(*), b1(*), a3(*), b3(*)

    real(dp) :: tmp_r, tmp_c

    tmp_r = div_r
    tmp_c = div_c
    div_r = tmp_r*b3(i3) + tmp_c*a3(i3)
    div_c = tmp_c*b3(i3) - tmp_r*a3(i3)
    if (i3 > n3/2 + 1) div_r = -div_r
    if (i3 > n3/2 + 1) div_c = -div_c

    tmp_r = div_r
    tmp_c = div_c
    div_r = tmp_r*b1(i1) + tmp_c*a1(i1)
    div_c = tmp_c*b1(i1) - tmp_r*a1(i1)
    if (i1 > n1/2 + 1) div_r = -div_r
    if (i1 > n1/2 + 1) div_c = -div_c

  end subroutine postprocess_dim13_fw

  attributes(device) subroutine postprocess_dim13_bw(div_r, div_c, i1, i3, &
                                                     n1, n3, a1, b1, a3, b3)
    !! Backward counterpart of postprocess_dim13_fw.
    implicit none

    real(dp), intent(inout) :: div_r, div_c
    integer, intent(in) :: i1, i3, n1, n3
    real(dp), intent(in) :: a1(*), b1(*), a3(*), b3(*)

    real(dp) :: tmp_r, tmp_c

    tmp_r = div_r
    tmp_c = div_c
    div_r = tmp_r*b3(i3) - tmp_c*a3(i3)
    div_c = tmp_c*b3(i3) + tmp_r*a3(i3)
    if (i3 > n3/2 + 1) div_r = -div_r
    if (i3 > n3/2 + 1) div_c = -div_c

    tmp_r = div_r
    tmp_c = div_c
    div_r = tmp_r*b1(i1) - tmp_c*a1(i1)
    div_c = tmp_c*b1(i1) + tmp_r*a1(i1)
    if (i1 > n1/2 + 1) div_r = -div_r
    if (i1 > n1/2 + 1) div_c = -div_c

  end subroutine postprocess_dim13_bw

  attributes(device) function pair_fw(own, partner, i2, a2, b2) result(res)
    !! Forward paired split of one dim2 mode against its mirror. The rule
    !! is symmetric, so the partner entry is pair_fw(partner, own, i2_rev).
    implicit none

    complex(dp), intent(in) :: own, partner
    integer, intent(in) :: i2
    real(dp), intent(in) :: a2(*), b2(*)
    complex(dp) :: res

    real(dp) :: l_r, l_c, r_r, r_c

    l_r = real(own, kind=dp)
    l_c = aimag(own)
    r_r = real(partner, kind=dp)
    r_c = aimag(partner)

    res = 0.5_dp*cmplx( & !&
      l_r*b2(i2) + l_c*a2(i2) + r_r*b2(i2) - r_c*a2(i2), &
      -l_r*a2(i2) + l_c*b2(i2) + r_r*a2(i2) + r_c*b2(i2), kind=dp &
      )

  end function pair_fw

  attributes(device) function pair_bw(own, partner, i2, a2, b2) result(res)
    !! Backward counterpart of pair_fw, symmetric in the same way.
    implicit none

    complex(dp), intent(in) :: own, partner
    integer, intent(in) :: i2
    real(dp), intent(in) :: a2(*), b2(*)
    complex(dp) :: res

    real(dp) :: l_r, l_c, r_r, r_c

    l_r = real(own, kind=dp)
    l_c = aimag(own)
    r_r = real(partner, kind=dp)
    r_c = aimag(partner)

    res = cmplx( & !&
      l_r*b2(i2) - l_c*a2(i2) + r_r*a2(i2) + r_c*b2(i2), &
      l_r*a2(i2) + l_c*b2(i2) - r_r*b2(i2) + r_c*a2(i2), kind=dp &
      )

  end function pair_bw

  attributes(device) subroutine poisson_divide(div_r, div_c, wave)
    !! Divide one entry by its spectral equivalence constant, with the
    !! eps_wave guard that keeps roundoff noise out of the pressure field.
    implicit none

    real(dp), intent(inout) :: div_r, div_c
    complex(dp), intent(in) :: wave

    real(dp) :: tmp_r, tmp_c

    tmp_r = real(wave, kind=dp)
    tmp_c = aimag(wave)
    if (abs(tmp_r) < eps_wave) then
      div_r = 0._dp
    else
      div_r = -div_r/tmp_r
    end if
    if (abs(tmp_c) < eps_wave) then
      div_c = 0._dp
    else
      div_c = -div_c/tmp_c
    end if

  end subroutine poisson_divide

  attributes(global) subroutine memcpy3D(dst, src, nx, ny, nz)
    !! Copy data between x3d2 padded arrays and cuFFTMp descriptors
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: dst
    real(dp), device, intent(in), dimension(:, :, :) :: src
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x !ny
    k = blockIdx%y !nz

    if (j <= ny) then
      do i = 1, nx
        dst(i, j, k) = src(i, j, k)
      end do
    end if
  end subroutine memcpy3D

  attributes(global) subroutine memcpy3D_with_transpose(dst, src, nx, ny, nz)
  !! Copy with transpose: src(nx, ny, nz) -> dst(ny, nx, nz)
  !! Used for 100 case forward FFT
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: dst  ! (ny+2, nx, nz) but we only write (ny, nx, nz)
    real(dp), device, intent(in), dimension(:, :, :) :: src   ! (nx, ny, nz)
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! iterates over nx
    k = blockIdx%y  ! nz

    if (i <= nx) then
      do j = 1, ny
        ! Transpose: dst(j, i, k) = src(i, j, k)
        dst(j, i, k) = src(i, j, k)
      end do
    end if

  end subroutine memcpy3D_with_transpose

  attributes(global) subroutine memcpy3D_with_transpose_back( &
    dst, src, nx, ny, nz &
    )
  !! Copy with transpose back: src(ny, nx, nz) -> dst(nx, ny, nz)
  !! Used for 100 case backward FFT
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: dst  ! (nx, ny, nz)
    real(dp), device, intent(in), dimension(:, :, :) :: src   ! (ny+2, nx, nz) but we only read (ny, nx, nz)
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! iterates over nx
    k = blockIdx%y  ! nz

    if (i <= nx) then
      do j = 1, ny
        ! Transpose back: dst(i, j, k) = src(j, i, k)
        dst(i, j, k) = src(j, i, k)
      end do
    end if

  end subroutine memcpy3D_with_transpose_back

  attributes(global) subroutine transpose_xyz_to_zxy(dst, src, nx, ny, nz)
    !! Transpose: src(nx, ny, nz) → dst(nz, nx, ny)
    !! Used before R2C FFT for 110 case to put Z (periodic) in fast dim.
    !!
    !! Launch: blocks=dim3(nz, (ny-1)/tpb+1, 1), threads=dim3(tpb,1,1)
    !! Each thread handles one (k,j) pair, loops over i.
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: dst  ! (nz, nx, ny)
    real(dp), device, intent(in), dimension(:, :, :) :: src   ! (nx, ny, nz)
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k

    k = blockIdx%x                                          ! nz
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x           ! ny

    if (j <= ny) then
      do i = 1, nx
        dst(k, i, j) = src(i, j, k)
      end do
    end if

  end subroutine transpose_xyz_to_zxy

  attributes(global) subroutine transpose_zxy_to_xyz(dst, src, nx, ny, nz)
    !! Transpose back: src(nz, nx, ny) -> dst(nx, ny, nz)
    !! Used after C2R FFT for 110 case.
    !!
    !! Launch: blocks=dim3(nz, (ny-1)/tpb+1, 1), threads=dim3(tpb,1,1)
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: dst  ! (nx, ny, nz)
    real(dp), device, intent(in), dimension(:, :, :) :: src   ! (nz, nx, ny)
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k

    k = blockIdx%x                                          ! nz
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x           ! ny

    if (j <= ny) then
      do i = 1, nx
        dst(i, j, k) = src(k, i, j)
      end do
    end if

  end subroutine transpose_zxy_to_xyz

  attributes(global) subroutine process_spectral_000( &
    div_u, waves, nx_spec, ny_spec, y_sp_st, nx, ny, nz, &
    ax, bx, ay, by, az, bz &
    )
    !! Post-processes the divergence of velocity in spectral space, including
    !! scaling w.r.t. grid size.
    !!
    !! Ref. JCP 228 (2009), 5989–6015, Sec 4
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    !> Spectral equivalence constants
    complex(dp), device, intent(in), dimension(:, :, :) :: waves
    real(dp), device, intent(in), dimension(:) :: ax, bx, ay, by, az, bz
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, ny_spec
    !> Offset in y direction in the permuted slabs in spectral space
    integer, value, intent(in) :: y_sp_st
    !> Grid size
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k, ix, iy, iz
    real(dp) :: tmp_r, tmp_c, div_r, div_c

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    if (j <= ny_spec) then
      do i = 1, nx_spec
        ! normalisation
        div_r = real(div_u(i, j, k), kind=dp)/nx/ny/nz
        div_c = aimag(div_u(i, j, k))/nx/ny/nz

        ix = i; iy = j + y_sp_st; iz = k

        ! post-process forward
        ! post-process in z
        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*bz(iz) + tmp_c*az(iz)
        div_c = tmp_c*bz(iz) - tmp_r*az(iz)
        if (iz > nz/2 + 1) div_r = -div_r
        if (iz > nz/2 + 1) div_c = -div_c

        ! post-process in y
        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*by(iy) + tmp_c*ay(iy)
        div_c = tmp_c*by(iy) - tmp_r*ay(iy)
        if (iy > ny/2 + 1) div_r = -div_r
        if (iy > ny/2 + 1) div_c = -div_c

        ! post-process in x
        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*bx(ix) + tmp_c*ax(ix)
        div_c = tmp_c*bx(ix) - tmp_r*ax(ix)

        ! Solve Poisson
        tmp_r = real(waves(i, j, k), kind=dp)
        tmp_c = aimag(waves(i, j, k))
        if ((tmp_r < eps_wave) .or. (tmp_c < eps_wave)) then
          div_r = 0._dp; div_c = 0._dp
        else
          div_r = -div_r/tmp_r
          div_c = -div_c/tmp_c
        end if

        ! post-process backward
        ! post-process in z
        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*bz(iz) - tmp_c*az(iz)
        div_c = -tmp_c*bz(iz) - tmp_r*az(iz)
        if (iz > nz/2 + 1) div_r = -div_r
        if (iz > nz/2 + 1) div_c = -div_c

        ! post-process in y
        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*by(iy) + tmp_c*ay(iy)
        div_c = tmp_c*by(iy) - tmp_r*ay(iy)
        if (iy > ny/2 + 1) div_r = -div_r
        if (iy > ny/2 + 1) div_c = -div_c

        ! post-process in x
        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*bx(ix) + tmp_c*ax(ix)
        div_c = -tmp_c*bx(ix) + tmp_r*ax(ix)

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_000

  attributes(global) subroutine process_spectral_010( &
    div_u, waves, nx_spec, ny_spec, y_sp_st, nx, ny, nz, &
    ax, bx, ay, by, az, bz &
    )
    !! Post-processes the divergence of velocity in spectral space, including
    !! scaling w.r.t. grid size.
    !!
    !! Ref. JCP 228 (2009), 5989–6015, Sec 4
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    !> Spectral equivalence constants
    complex(dp), device, intent(in), dimension(:, :, :) :: waves
    real(dp), device, intent(in), dimension(:) :: ax, bx, ay, by, az, bz
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, ny_spec
    !> Offset in y direction in the permuted slabs in spectral space
    integer, value, intent(in) :: y_sp_st
    !> Grid size
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k, ix, iy, iz, iy_rev
    real(dp) :: div_r, div_c
    complex(dp) :: own, partner

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    if (i <= nx_spec) then
      do j = 1, ny_spec
        ix = i; iz = k

        ! normalisation
        div_r = real(div_u(i, j, k), kind=dp)/nx/ny/nz
        div_c = aimag(div_u(i, j, k))/nx/ny/nz

        call postprocess_dim13_fw(div_r, div_c, ix, iz, nx, nz, &
                                  ax, bx, az, bz)

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

    if (i <= nx_spec) then
      do j = 2, ny_spec/2 + 1
        iy = j + y_sp_st
        iy_rev = ny_spec - j + 2 + y_sp_st

        ! both entries are read before either is written
        own = div_u(i, j, k)
        partner = div_u(i, ny_spec - j + 2, k)

        ! update the entry
        div_u(i, j, k) = pair_fw(own, partner, iy, ay, by)
        div_u(i, ny_spec - j + 2, k) = pair_fw(partner, own, iy_rev, ay, by)
      end do
    end if

    ! Solve Poisson
    if (i <= nx_spec) then
      do j = 1, ny_spec
        div_r = real(div_u(i, j, k), kind=dp)
        div_c = aimag(div_u(i, j, k))

        call poisson_divide(div_r, div_c, waves(i, j, k))

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
        if (i == nx/2 + 1 .and. k == nz/2 + 1) div_u(i, j, k) = 0._dp
      end do
    end if

    ! post-process backward
    if (i <= nx_spec) then
      do j = 2, ny_spec/2 + 1
        iy = j + y_sp_st
        iy_rev = ny_spec - j + 2 + y_sp_st

        own = div_u(i, j, k)
        partner = div_u(i, ny_spec - j + 2, k)

        ! update the entry
        div_u(i, j, k) = pair_bw(own, partner, iy, ay, by)
        div_u(i, ny_spec - j + 2, k) = pair_bw(partner, own, iy_rev, ay, by)
      end do
    end if

    if (i <= nx_spec) then
      do j = 1, ny_spec
        ix = i; iz = k

        div_r = real(div_u(i, j, k), kind=dp)
        div_c = aimag(div_u(i, j, k))

        call postprocess_dim13_bw(div_r, div_c, ix, iz, nx, nz, &
                                  ax, bx, az, bz)

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_010

  attributes(global) subroutine process_spectral_010_fw( &
    div_u, nx_spec, ny_spec, y_sp_st, z_off, nx, ny, nz, ax, bx, ay, by, &
    az, bz &
    )
    !! Post-processes the divergence of velocity in spectral space, including
    !! scaling w.r.t. grid size.
    !!
    !! Ref. JCP 228 (2009), 5989–6015, Sec 4
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    !> Spectral equivalence constants
    real(dp), device, intent(in), dimension(:) :: ax, bx, ay, by, az, bz
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, ny_spec
    !> Offset in y direction in the permuted slabs in spectral space
    integer, value, intent(in) :: y_sp_st
    !> Offset in z direction of the block div_u holds
    integer, value, intent(in) :: z_off
    !> Grid size
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k, ix, iy, iz, iy_rev
    real(dp) :: div_r, div_c
    complex(dp) :: own, partner

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    if (i <= nx_spec) then
      do j = 1, ny_spec
        ix = i; iz = k + z_off

        ! normalisation
        div_r = real(div_u(i, j, k), kind=dp)/nx/ny/nz
        div_c = aimag(div_u(i, j, k))/nx/ny/nz

        call postprocess_dim13_fw(div_r, div_c, ix, iz, nx, nz, &
                                  ax, bx, az, bz)

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

    if (i <= nx_spec) then
      do j = 2, ny_spec/2 + 1
        iy = j + y_sp_st
        iy_rev = ny_spec - j + 2 + y_sp_st

        own = div_u(i, j, k)
        partner = div_u(i, ny_spec - j + 2, k)

        ! update the entry
        div_u(i, j, k) = pair_fw(own, partner, iy, ay, by)
        div_u(i, ny_spec - j + 2, k) = pair_fw(partner, own, iy_rev, ay, by)
      end do
    end if

  end subroutine process_spectral_010_fw

  attributes(global) subroutine process_spectral_010_poisson( &
    div_u, a_re, a_im, off, inc, nx_spec, n, z_off, nx, ny, nz &
    )
    !! Solve the Poisson equation at cell centres with non-periodic BC along y
    !!
    !! Ref. JCP 228 (2009), 5989–6015, Sec 4
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    !> Spectral equivalence constants
    real(dp), device, intent(inout), dimension(:, :, :, :) :: a_re, a_im
    !> offset and increment. increment is 2 when considering only odd or even
    integer, value, intent(in) :: off, inc
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, n, nx, ny, nz
    !> Offset in z direction of the block div_u holds
    integer, value, intent(in) :: z_off

    integer :: i, j, k, jm, nm
    real(dp) :: tmp_r, tmp_c, div_r, div_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    ! Solve Poisson
    if (i <= nx_spec) then
      ! Forward pass for the pentadiagonal matrix
      do j = 1, n - 2
        ! j mapping based on odd/even
        ! inc=2, off=0 ---> j => 2j - 1
        ! inc=2, off=1 ---> j => 2j
        ! inc=1, off=0 ---> j => j
        jm = inc*j + off - inc/2
        ! eliminate diag-1
        tmp_r = 0._dp
        if (abs(a_re(i, j, k, 3)) > eps_wave) then
          tmp_r = a_re(i, j + 1, k, 2)/a_re(i, j, k, 3)
        end if
        tmp_c = 0._dp
        if (abs(a_im(i, j, k, 3)) > eps_wave) then
          tmp_c = a_im(i, j + 1, k, 2)/a_im(i, j, k, 3)
        end if
        div_r = real(div_u(i, jm + inc, k) - tmp_r*div_u(i, jm, k), kind=dp)
        div_c = aimag(div_u(i, jm + inc, k) - tmp_c*div_u(i, jm, k))
        div_u(i, jm + inc, k) = cmplx(div_r, div_c, kind=dp)
        ! modify pentadiagonal coefficients in-place
        a_re(i, j + 1, k, 3) = a_re(i, j + 1, k, 3) - tmp_r*a_re(i, j, k, 4)
        a_im(i, j + 1, k, 3) = a_im(i, j + 1, k, 3) - tmp_c*a_im(i, j, k, 4)
        a_re(i, j + 1, k, 4) = a_re(i, j + 1, k, 4) - tmp_r*a_re(i, j, k, 5)
        a_im(i, j + 1, k, 4) = a_im(i, j + 1, k, 4) - tmp_c*a_im(i, j, k, 5)

        ! eliminate diag-2
        tmp_r = 0._dp
        if (abs(a_re(i, j, k, 3)) > eps_wave) then
          tmp_r = a_re(i, j + 2, k, 1)/a_re(i, j, k, 3)
        end if
        tmp_c = 0._dp
        if (abs(a_im(i, j, k, 3)) > eps_wave) then
          tmp_c = a_im(i, j + 2, k, 1)/a_im(i, j, k, 3)
        end if
        div_r = real(div_u(i, jm + 2*inc, k) - tmp_r*div_u(i, jm, k), kind=dp)
        div_c = aimag(div_u(i, jm + 2*inc, k) - tmp_c*div_u(i, jm, k))
        div_u(i, jm + 2*inc, k) = cmplx(div_r, div_c, kind=dp)
        ! modify pentadiagonal coefficients in-place
        a_re(i, j + 2, k, 2) = a_re(i, j + 2, k, 2) - tmp_r*a_re(i, j, k, 4)
        a_im(i, j + 2, k, 2) = a_im(i, j + 2, k, 2) - tmp_c*a_im(i, j, k, 4)
        a_re(i, j + 2, k, 3) = a_re(i, j + 2, k, 3) - tmp_r*a_re(i, j, k, 5)
        a_im(i, j + 2, k, 3) = a_im(i, j + 2, k, 3) - tmp_c*a_im(i, j, k, 5)
      end do

      ! handle the last row
      if (abs(a_re(i, n - 1, k, 3)) > eps_wave) then
        tmp_r = a_re(i, n, k, 2)/a_re(i, n - 1, k, 3)
      else
        tmp_r = 0._dp
      end if
      if (abs(a_im(i, n - 1, k, 3)) > eps_wave) then
        tmp_c = a_im(i, n, k, 2)/a_im(i, n - 1, k, 3)
      else
        tmp_c = 0._dp
      end if
      div_r = a_re(i, n, k, 3) - tmp_r*a_re(i, n - 1, k, 4)
      div_c = a_im(i, n, k, 3) - tmp_c*a_im(i, n - 1, k, 4)

      ! j mapping based on odd/even for last point j=n
      nm = inc*n + off - inc/2
      if (abs(div_r) > eps_wave) then
        tmp_r = tmp_r/div_r
        div_r = real(div_u(i, nm, k), kind=dp)/div_r &
                - tmp_r*real(div_u(i, nm - inc, k), kind=dp)
      else
        tmp_r = 0._dp
        div_r = 0._dp
      end if
      if (abs(div_c) > eps_wave) then
        tmp_c = tmp_c/div_c
        div_c = aimag(div_u(i, nm, k))/div_c &
                - tmp_c*aimag(div_u(i, nm - inc, k))
      else
        tmp_c = 0._dp
        div_c = 0._dp
      end if
      div_u(i, nm, k) = cmplx(div_r, div_c, kind=dp)

      if (abs(a_re(i, n - 1, k, 3)) > eps_wave) then
        tmp_r = 1._dp/a_re(i, n - 1, k, 3)
      else
        tmp_r = 0._dp
      end if
      if (abs(a_im(i, n - 1, k, 3)) > eps_wave) then
        tmp_c = 1._dp/a_im(i, n - 1, k, 3)
      else
        tmp_c = 0._dp
      end if
      div_r = a_re(i, n - 1, k, 4)*tmp_r
      div_c = a_im(i, n - 1, k, 4)*tmp_c
      div_u(i, nm - inc, k) = cmplx( & !&
        real(div_u(i, nm - inc, k), kind=dp)*tmp_r &
        - real(div_u(i, nm, k), kind=dp)*div_r, &
        aimag(div_u(i, nm - inc, k))*tmp_c &
        - aimag(div_u(i, nm, k))*div_c, &
        kind=dp &
        )

      if (i == nx/2 + 1 .and. k + z_off == nz/2 + 1) then
        div_u(i, nm, k) = 0._dp
        div_u(i, nm - inc, k) = 0._dp
      end if

      ! backward pass
      do j = n - 2, 1, -1
        ! j mapping based on odd/even
        jm = inc*j + off - inc/2
        if (abs(a_re(i, j, k, 3)) > eps_wave) then
          tmp_r = 1._dp/a_re(i, j, k, 3)
        else
          tmp_r = 0._dp
        end if
        if (abs(a_im(i, j, k, 3)) > eps_wave) then
          tmp_c = 1._dp/a_im(i, j, k, 3)
        else
          tmp_c = 0._dp
        end if
        div_u(i, jm, k) = cmplx( & !&
          tmp_r*(real(div_u(i, jm, k), kind=dp) &
                 - a_re(i, j, k, 4)*real(div_u(i, jm + inc, k), kind=dp) &
                 - a_re(i, j, k, 5)*real(div_u(i, jm + 2*inc, k), kind=dp)), &
          tmp_c*(aimag(div_u(i, jm, k)) &
                 - a_im(i, j, k, 4)*aimag(div_u(i, jm + inc, k)) &
                 - a_im(i, j, k, 5)*aimag(div_u(i, jm + 2*inc, k))), &
          kind=dp &
          )
        if (i == nx/2 + 1 .and. k + z_off == nz/2 + 1) div_u(i, jm, k) = 0._dp
      end do
    end if

  end subroutine process_spectral_010_poisson

  attributes(global) subroutine process_spectral_010_bw( &
    div_u, nx_spec, ny_spec, y_sp_st, z_off, nx, ny, nz, ax, bx, ay, by, &
    az, bz &
    )
    !! Post-processes the divergence of velocity in spectral space, including
    !! scaling w.r.t. grid size.
    !!
    !! Ref. JCP 228 (2009), 5989–6015, Sec 4
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    !> Spectral equivalence constants
    real(dp), device, intent(in), dimension(:) :: ax, bx, ay, by, az, bz
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, ny_spec
    !> Offset in y direction in the permuted slabs in spectral space
    integer, value, intent(in) :: y_sp_st
    !> Offset in z direction of the block div_u holds
    integer, value, intent(in) :: z_off
    !> Grid size
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k, ix, iy, iz, iy_rev
    real(dp) :: div_r, div_c
    complex(dp) :: own, partner

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    ! post-process backward
    if (i <= nx_spec) then
      do j = 2, ny_spec/2 + 1
        iy = j + y_sp_st
        iy_rev = ny_spec - j + 2 + y_sp_st

        own = div_u(i, j, k)
        partner = div_u(i, ny_spec - j + 2, k)

        ! update the entry
        div_u(i, j, k) = pair_bw(own, partner, iy, ay, by)
        div_u(i, ny_spec - j + 2, k) = pair_bw(partner, own, iy_rev, ay, by)
      end do
    end if

    if (i <= nx_spec) then
      do j = 1, ny_spec
        ix = i; iz = k + z_off

        div_r = real(div_u(i, j, k), kind=dp)
        div_c = aimag(div_u(i, j, k))

        call postprocess_dim13_bw(div_r, div_c, ix, iz, nx, nz, &
                                  ax, bx, az, bz)

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_010_bw

  ! ------------------------------------------------------------------
  ! STAGED DIM2 MIRROR SPECTRAL KERNELS, MULTIPLE RANKS (3 staged launches)
  !
  ! Staged form of the dim2 mirror postprocess, shared by the 010 and
  ! 100 cases. The paired split couples mode j with mode n - j + 2, so
  ! once dim2 is distributed over ranks the partner value sits on
  ! another GPU. Splitting the single kernel into three launches leaves
  ! room for a mirror exchange either side of the Poisson solve, and the
  ! pairing then reads its partner from that mirror instead of from
  ! div_u itself.
  !
  ! Every rank updates only the modes it owns. The 010 pairing rule is
  ! symmetric, new(iy) = f(own(iy), partner(n - iy + 2), iy), so nothing
  ! has to be sent back afterwards.
  !
  ! mirror holds the partner rank's slab and is read in reverse as
  ! mirror(i, ny_spec - j + 2, k). Local j = 1 maps one plane past the
  ! end of that slab and is supplied separately in mirror_plane(i, k).
  ! Global mode 1 is self paired and is left alone, exactly as the 010
  ! loop starting at j = 2 leaves it.
  !
  ! These keep the dim1/dim2 naming of the 010 kernels rather than the
  ! physical directions. dim1 is the periodic R2C direction and dim2 is
  ! the non-periodic one. For 100 dim1 = y and dim2 = x, for 010
  ! dim1 = x and dim2 = y, and dim3 = z in both.
  ! ------------------------------------------------------------------

  attributes(global) subroutine process_spectral_fw( &
    div_u, nx_spec, ny_spec, nx, ny, nz, ax, bx, az, bz &
    )
    !! Stage 1 (forward): normalisation followed by the dim3 and dim1
    !! postprocess. Every mode is handled on its own here.
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    real(dp), device, intent(in), dimension(:) :: ax, bx, az, bz
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, ny_spec
    !> Grid size
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k, ix, iz
    real(dp) :: div_r, div_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    if (i <= nx_spec) then
      do j = 1, ny_spec
        ix = i; iz = k

        ! normalisation
        div_r = real(div_u(i, j, k), kind=dp)/nx/ny/nz
        div_c = aimag(div_u(i, j, k))/nx/ny/nz

        call postprocess_dim13_fw(div_r, div_c, ix, iz, nx, nz, &
                                  ax, bx, az, bz)

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_fw

  attributes(global) subroutine process_spectral_pair_fw( &
    div_u, mirror, mirror_plane, waves, nx_spec, ny_spec, y_sp_st, &
    nx, nz, ay, by &
    )
    !! Stage 2 (forward): the dim2 paired split, then the Poisson solve.
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    !> dim2 mirror of the local slab, and the single plane below it
    complex(dp), device, intent(in), dimension(:, :, :) :: mirror
    complex(dp), device, intent(in), dimension(:, :) :: mirror_plane
    !> Spectral equivalence constants
    complex(dp), device, intent(in), dimension(:, :, :) :: waves
    real(dp), device, intent(in), dimension(:) :: ay, by
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, ny_spec
    !> Offset in the paired direction in the permuted slabs in spectral space
    integer, value, intent(in) :: y_sp_st
    !> Grid size
    integer, value, intent(in) :: nx, nz

    integer :: i, j, k, iy
    real(dp) :: div_r, div_c
    complex(dp) :: partner

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    if (i <= nx_spec) then
      do j = 1, ny_spec
        iy = j + y_sp_st

        ! global mode 1 is self paired, the 010 loop never touches it
        if (iy /= 1) then
          if (j == 1) then
            partner = mirror_plane(i, k)
          else
            partner = mirror(i, ny_spec - j + 2, k)
          end if

          ! update the entry, only this rank's own mode
          div_u(i, j, k) = pair_fw(div_u(i, j, k), partner, iy, ay, by)
        end if
      end do
    end if

    ! Solve Poisson
    if (i <= nx_spec) then
      do j = 1, ny_spec
        div_r = real(div_u(i, j, k), kind=dp)
        div_c = aimag(div_u(i, j, k))

        call poisson_divide(div_r, div_c, waves(i, j, k))

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
        if (i == nx/2 + 1 .and. k == nz/2 + 1) div_u(i, j, k) = 0._dp
      end do
    end if

  end subroutine process_spectral_pair_fw

  attributes(global) subroutine process_spectral_pair_bw( &
    div_u, mirror, mirror_plane, nx_spec, ny_spec, y_sp_st, &
    nx, nz, ax, bx, ay, by, az, bz &
    )
    !! Stage 3 (backward): the dim2 paired recombination, then the dim3
    !! and dim1 postprocess.
    implicit none

    !> Divergence of velocity in spectral space
    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u
    !> dim2 mirror of the local slab, and the single plane below it
    complex(dp), device, intent(in), dimension(:, :, :) :: mirror
    complex(dp), device, intent(in), dimension(:, :) :: mirror_plane
    real(dp), device, intent(in), dimension(:) :: ax, bx, ay, by, az, bz
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, ny_spec
    !> Offset in the paired direction in the permuted slabs in spectral space
    integer, value, intent(in) :: y_sp_st
    !> Grid size
    integer, value, intent(in) :: nx, nz

    integer :: i, j, k, ix, iy, iz
    real(dp) :: div_r, div_c
    complex(dp) :: partner

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    ! postprocess backward
    if (i <= nx_spec) then
      do j = 1, ny_spec
        iy = j + y_sp_st

        ! global mode 1 is self paired, the 010 loop never touches it
        if (iy /= 1) then
          if (j == 1) then
            partner = mirror_plane(i, k)
          else
            partner = mirror(i, ny_spec - j + 2, k)
          end if

          ! update the entry, only this rank's own mode
          div_u(i, j, k) = pair_bw(div_u(i, j, k), partner, iy, ay, by)
        end if
      end do
    end if

    if (i <= nx_spec) then
      do j = 1, ny_spec
        ix = i; iz = k

        div_r = real(div_u(i, j, k), kind=dp)
        div_c = aimag(div_u(i, j, k))

        call postprocess_dim13_bw(div_r, div_c, ix, iz, nx, nz, &
                                  ax, bx, az, bz)

        ! update the entry
        div_u(i, j, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_pair_bw

  attributes(global) subroutine pack_spectral_plane( &
    plane, div_u, nx_spec, nz_spec &
    )
    !! Copies the first dim2 plane of the local spectral slab into a
    !! contiguous buffer so it can be handed to MPI.
    implicit none

    complex(dp), device, intent(out), dimension(:, :) :: plane
    complex(dp), device, intent(in), dimension(:, :, :) :: div_u
    !> Grid size in spectral space
    integer, value, intent(in) :: nx_spec, nz_spec

    integer :: i, k

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y ! nz_spec

    if (i <= nx_spec .and. k <= nz_spec) plane(i, k) = div_u(i, 1, k)

  end subroutine pack_spectral_plane

! ------------------------------------------------------------------
  ! 110 SPECTRAL KERNELS (7 separate launches)
  !
  ! Spectral array layout: (nz/2+1, nx, ny) — Z R2C in dim1
  ! Thread layout:
  !   i = threadIdx → X (dim2, up to nx)
  !   k = blockIdx%y → Y (dim3, up to ny)
  !   serial loop j → Z R2C modes (dim1, nz/2+1)
  !
  ! Launch config for all:
  !   blocks = dim3((nx-1)/tsize+1, ny, 1)
  !   threads = dim3(min(nx, tsize), 1, 1)
  !
  ! Arguments use PHYSICAL names (nx,ny,nz) not spectral-dim names,
  ! to avoid confusion with the transposed layout.
  ! ------------------------------------------------------------------

  attributes(global) subroutine process_spectral_110_norm_z( &
    div_u, nz_h, nx, ny, nz, nx_glob, ny_glob, az, bz &
    )
    !! Step 1 (forward): normalise + Z periodic post-process
    !! Z is dim1 (serial j loop), periodic R2C — no sign flip needed
    !! since j only goes to nz/2+1.
    !! nx, ny are the local loop bounds (equal to nx_glob, ny_glob on a
    !! single rank); the normalisation divisor always needs the globals.
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx, ny)
    real(dp), device, intent(in), dimension(:) :: az, bz
    integer, value, intent(in) :: nz_h  ! nz/2+1
    integer, value, intent(in) :: nx, ny, nz
    integer, value, intent(in) :: nx_glob, ny_glob

    integer :: i, j, k
    real(dp) :: tmp_r, tmp_c, div_r, div_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! X index
    k = blockIdx%y                                   ! Y index

    if (i <= nx .and. k <= ny) then
      do j = 1, nz_h
        div_r = real(div_u(j, i, k), kind=dp)/nx_glob/ny_glob/nz
        div_c = aimag(div_u(j, i, k))/nx_glob/ny_glob/nz

        ! Z periodic post-process (forward)
        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*bz(j) + tmp_c*az(j)
        div_c = tmp_c*bz(j) - tmp_r*az(j)
        ! No sign flip: j <= nz/2+1 always, so j > nz/2+1 is never true

        div_u(j, i, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_110_norm_z

  attributes(global) subroutine process_spectral_110_x_pair_fw( &
    div_u, nz_h, nx, ny, x_sp_st, ax, bx &
    )
    !! Step 2 (forward): X paired even/odd split
    !! X is dim2 (thread i). Only i in [2, nx/2+1] executes.
    !! Writes to i and nx-i+2. Race-free (pair doesn't enter).
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx, ny)
    real(dp), device, intent(in), dimension(:) :: ax, bx
    integer, value, intent(in) :: nz_h, nx, ny
    integer, value, intent(in) :: x_sp_st

    integer :: i, j, k, ix, ix_pair
    real(dp) :: l_r, l_c, r_r, r_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! X index
    k = blockIdx%y                                   ! Y index
    ! Note: when i == nx/2+1 (even nx), ix_pair == i (Nyquist self-pair).
    ! Both writes produce the same result, second is a harmless overwrite.
    if (i >= 2 .and. i <= nx/2 + 1 .and. k <= ny) then
      ix = i + x_sp_st
      ix_pair = nx - i + 2
      do j = 1, nz_h
        l_r = real(div_u(j, i, k), kind=dp)
        l_c = aimag(div_u(j, i, k))
        r_r = real(div_u(j, ix_pair, k), kind=dp)
        r_c = aimag(div_u(j, ix_pair, k))

        div_u(j, i, k) = 0.5_dp*cmplx( & !&
          l_r*bx(ix) + l_c*ax(ix) + r_r*bx(ix) - r_c*ax(ix), &
          -l_r*ax(ix) + l_c*bx(ix) + r_r*ax(ix) + r_c*bx(ix), kind=dp &
          )
        div_u(j, ix_pair, k) = 0.5_dp*cmplx( & !&
          r_r*bx(ix_pair + x_sp_st) + r_c*ax(ix_pair + x_sp_st) &
            + l_r*bx(ix_pair + x_sp_st) - l_c*ax(ix_pair + x_sp_st), &
          -r_r*ax(ix_pair + x_sp_st) + r_c*bx(ix_pair + x_sp_st) &
            + l_r*ax(ix_pair + x_sp_st) + l_c*bx(ix_pair + x_sp_st), &
          kind=dp &
          )
      end do
    end if

  end subroutine process_spectral_110_x_pair_fw

  attributes(global) subroutine process_spectral_110_y_pair_fw( &
    div_u, nz_h, nx, ny, y_sp_st, ay, by &
    )
    !! Step 3 (forward): Y paired even/odd split
    !! Y is dim3 (blockIdx%y = k). Only k in [2, ny/2+1] executes.
    !! Writes to k and ny-k+2. Race-free (pair block doesn't enter).
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx, ny)
    real(dp), device, intent(in), dimension(:) :: ay, by
    integer, value, intent(in) :: nz_h, nx, ny
    integer, value, intent(in) :: y_sp_st

    integer :: i, j, k, iy, iy_pair
    real(dp) :: l_r, l_c, r_r, r_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! X index
    k = blockIdx%y                                   ! Y index

    if (i <= nx .and. k >= 2 .and. k <= ny/2 + 1) then
      iy = k + y_sp_st
      iy_pair = ny - k + 2
      do j = 1, nz_h
        l_r = real(div_u(j, i, k), kind=dp)
        l_c = aimag(div_u(j, i, k))
        r_r = real(div_u(j, i, iy_pair), kind=dp)
        r_c = aimag(div_u(j, i, iy_pair))

        div_u(j, i, k) = 0.5_dp*cmplx( & !&
          l_r*by(iy) + l_c*ay(iy) + r_r*by(iy) - r_c*ay(iy), &
          -l_r*ay(iy) + l_c*by(iy) + r_r*ay(iy) + r_c*by(iy), kind=dp &
          )
        div_u(j, i, iy_pair) = 0.5_dp*cmplx( & !&
          r_r*by(iy_pair + y_sp_st) + r_c*ay(iy_pair + y_sp_st) &
            + l_r*by(iy_pair + y_sp_st) - l_c*ay(iy_pair + y_sp_st), &
          -r_r*ay(iy_pair + y_sp_st) + r_c*by(iy_pair + y_sp_st) &
            + l_r*ay(iy_pair + y_sp_st) + l_c*by(iy_pair + y_sp_st), &
          kind=dp &
          )
      end do
    end if

  end subroutine process_spectral_110_y_pair_fw

  attributes(global) subroutine process_spectral_110_poisson( &
    div_u, waves, nz_h, nx, ny &
    )
    !! Step 4: Poisson solve — divide by waves
    !! x and y are cosine transformed, so neither has a Nyquist mode and the
    !! only singular entry is the mean, which the eps_wave test handles. The
    !! x Nyquist zeroing of the 010 case must not be copied here: in this
    !! layout x index nx/2 + 1 is the regular cosine mode N/2 (#465).
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx, ny)
    complex(dp), device, intent(in), dimension(:, :, :) :: waves    ! (nz/2+1, nx, ny)
    integer, value, intent(in) :: nz_h, nx, ny

    integer :: i, j, k
    real(dp) :: div_r, div_c, tmp_r, tmp_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! X index
    k = blockIdx%y                                   ! Y index

    if (i <= nx .and. k <= ny) then
      do j = 1, nz_h
        div_r = real(div_u(j, i, k), kind=dp)
        div_c = aimag(div_u(j, i, k))

        tmp_r = real(waves(j, i, k), kind=dp)
        tmp_c = aimag(waves(j, i, k))
        if (abs(tmp_r) < eps_wave) then
          div_r = 0._dp
        else
          div_r = -div_r/tmp_r
        end if
        if (abs(tmp_c) < eps_wave) then
          div_c = 0._dp
        else
          div_c = -div_c/tmp_c
        end if

        div_u(j, i, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_110_poisson

  attributes(global) subroutine process_spectral_110_y_pair_bw( &
    div_u, nz_h, nx, ny, y_sp_st, ay, by &
    )
    !! Step 5 (backward): Y paired even/odd recombine
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx, ny)
    real(dp), device, intent(in), dimension(:) :: ay, by
    integer, value, intent(in) :: nz_h, nx, ny
    integer, value, intent(in) :: y_sp_st

    integer :: i, j, k, iy, iy_pair
    real(dp) :: l_r, l_c, r_r, r_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y

    if (i <= nx .and. k >= 2 .and. k <= ny/2 + 1) then
      iy = k + y_sp_st
      iy_pair = ny - k + 2
      do j = 1, nz_h
        l_r = real(div_u(j, i, k), kind=dp)
        l_c = aimag(div_u(j, i, k))
        r_r = real(div_u(j, i, iy_pair), kind=dp)
        r_c = aimag(div_u(j, i, iy_pair))

        div_u(j, i, k) = cmplx( & !&
          l_r*by(iy) - l_c*ay(iy) + r_r*ay(iy) + r_c*by(iy), &
          l_r*ay(iy) + l_c*by(iy) - r_r*by(iy) + r_c*ay(iy), kind=dp &
          )
        div_u(j, i, iy_pair) = cmplx( & !&
          r_r*by(iy_pair + y_sp_st) - r_c*ay(iy_pair + y_sp_st) &
            + l_r*ay(iy_pair + y_sp_st) + l_c*by(iy_pair + y_sp_st), &
          r_r*ay(iy_pair + y_sp_st) + r_c*by(iy_pair + y_sp_st) &
            - l_r*by(iy_pair + y_sp_st) + l_c*ay(iy_pair + y_sp_st), &
          kind=dp &
          )
      end do
    end if

  end subroutine process_spectral_110_y_pair_bw

  attributes(global) subroutine process_spectral_110_x_pair_bw( &
    div_u, nz_h, nx, ny, x_sp_st, ax, bx &
    )
    !! Step 6 (backward): X paired even/odd recombine
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx, ny)
    real(dp), device, intent(in), dimension(:) :: ax, bx
    integer, value, intent(in) :: nz_h, nx, ny
    integer, value, intent(in) :: x_sp_st

    integer :: i, j, k, ix, ix_pair
    real(dp) :: l_r, l_c, r_r, r_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y

    if (i >= 2 .and. i <= nx/2 + 1 .and. k <= ny) then
      ix = i + x_sp_st
      ix_pair = nx - i + 2
      do j = 1, nz_h
        l_r = real(div_u(j, i, k), kind=dp)
        l_c = aimag(div_u(j, i, k))
        r_r = real(div_u(j, ix_pair, k), kind=dp)
        r_c = aimag(div_u(j, ix_pair, k))

        div_u(j, i, k) = cmplx( & !&
          l_r*bx(ix) - l_c*ax(ix) + r_r*ax(ix) + r_c*bx(ix), &
          l_r*ax(ix) + l_c*bx(ix) - r_r*bx(ix) + r_c*ax(ix), kind=dp &
          )
        div_u(j, ix_pair, k) = cmplx( & !&
          r_r*bx(ix_pair + x_sp_st) - r_c*ax(ix_pair + x_sp_st) &
            + l_r*ax(ix_pair + x_sp_st) + l_c*bx(ix_pair + x_sp_st), &
          r_r*ax(ix_pair + x_sp_st) + r_c*bx(ix_pair + x_sp_st) &
            - l_r*bx(ix_pair + x_sp_st) + l_c*ax(ix_pair + x_sp_st), &
          kind=dp &
          )
      end do
    end if

  end subroutine process_spectral_110_x_pair_bw

  attributes(global) subroutine process_spectral_110_x_pair_fw_mirror( &
    div_u, mirror, plane, nz_h, nx_l, ny, x_sp_st, ax, bx &
    )
    !! Multi-rank counterpart of process_spectral_110_x_pair_fw: the X
    !! paired split, but the partner mode lives on another rank, reached
    !! through the dim2 mirror exchange (see exchange_dim2_mirror). Unlike
    !! the single-rank kernel this only updates this rank's own modes;
    !! the partner rank updates its own copy the same way.
    !! At the global Nyquist mode (even grid size) the mirror
    !! partner is this rank's own pre-update value, so the formula below
    !! self-pairs it exactly as the single-rank kernel does.
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx_l, ny)
    complex(dp), device, intent(in), dimension(:, :, :) :: mirror ! dim2 mirror of div_u
    complex(dp), device, intent(in), dimension(:, :) :: plane ! extra mirror plane
    real(dp), device, intent(in), dimension(:) :: ax, bx
    integer, value, intent(in) :: nz_h, nx_l, ny
    integer, value, intent(in) :: x_sp_st

    integer :: i, j, k, ix
    complex(dp) :: partner

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! local X index
    k = blockIdx%y                                   ! Y index

    if (i <= nx_l .and. k <= ny) then
      ix = i + x_sp_st

      ! global mode 1 is self paired, never touched here
      if (ix /= 1) then
        do j = 1, nz_h
          if (i == 1) then
            partner = plane(j, k)
          else
            partner = mirror(j, nx_l - i + 2, k)
          end if

          div_u(j, i, k) = pair_fw(div_u(j, i, k), partner, ix, ax, bx)
        end do
      end if
    end if

  end subroutine process_spectral_110_x_pair_fw_mirror

  attributes(global) subroutine process_spectral_110_x_pair_bw_mirror( &
    div_u, mirror, plane, nz_h, nx_l, ny, x_sp_st, ax, bx &
    )
    !! Multi-rank counterpart of process_spectral_110_x_pair_bw, see
    !! process_spectral_110_x_pair_fw_mirror.
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx_l, ny)
    complex(dp), device, intent(in), dimension(:, :, :) :: mirror ! dim2 mirror of div_u
    complex(dp), device, intent(in), dimension(:, :) :: plane ! extra mirror plane
    real(dp), device, intent(in), dimension(:) :: ax, bx
    integer, value, intent(in) :: nz_h, nx_l, ny
    integer, value, intent(in) :: x_sp_st

    integer :: i, j, k, ix
    complex(dp) :: partner

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y

    if (i <= nx_l .and. k <= ny) then
      ix = i + x_sp_st

      if (ix /= 1) then
        do j = 1, nz_h
          if (i == 1) then
            partner = plane(j, k)
          else
            partner = mirror(j, nx_l - i + 2, k)
          end if

          div_u(j, i, k) = pair_bw(div_u(j, i, k), partner, ix, ax, bx)
        end do
      end if
    end if

  end subroutine process_spectral_110_x_pair_bw_mirror

  attributes(global) subroutine process_spectral_110_z_bw( &
    div_u, nz_h, nx, ny, az, bz &
    )
    !! Step 7 (backward): Z periodic undo
    implicit none

    complex(dp), device, intent(inout), dimension(:, :, :) :: div_u ! (nz/2+1, nx, ny)
    real(dp), device, intent(in), dimension(:) :: az, bz
    integer, value, intent(in) :: nz_h, nx, ny

    integer :: i, j, k
    real(dp) :: tmp_r, tmp_c, div_r, div_c

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    k = blockIdx%y

    if (i <= nx .and. k <= ny) then
      do j = 1, nz_h
        div_r = real(div_u(j, i, k), kind=dp)
        div_c = aimag(div_u(j, i, k))

        tmp_r = div_r
        tmp_c = div_c
        div_r = tmp_r*bz(j) - tmp_c*az(j)
        div_c = tmp_c*bz(j) + tmp_r*az(j)
        ! No sign flip: j <= nz/2+1 always

        div_u(j, i, k) = cmplx(div_r, div_c, kind=dp)
      end do
    end if

  end subroutine process_spectral_110_z_bw

  ! ------------------------------------------------------------------
  ! 110 MULTI-RANK SLAB REDISTRIBUTION
  !
  ! The 110 physical domain is split along Z (nz_l = nz/nproc per rank),
  ! but the cuFFTMp input slab for the (nz, nx, ny) plan is split along Y
  ! (ny_l = ny/nproc per rank). These four kernels pack and unpack the
  ! MPI_Alltoall buffer (nz_l, nx, ny_l, nproc) that carries a rank's data
  ! between the two layouts; rank index s (0-based) is stored as dim4.
  !
  ! Launch config for all four: blocks = dim3(nz_l, (ny_l-1)/tpb+1, 1),
  ! threads = dim3(min(ny_l, tpb), 1, 1), modelled on transpose_xyz_to_zxy.
  ! ------------------------------------------------------------------

  attributes(global) subroutine pack_zslab_to_alltoall( &
    sendbuf, src, nx, ny_l, nz_l, nproc &
    )
    !! Packs the physical Z-slab field block (nx, ny, nz_l) into the
    !! MPI_Alltoall send buffer, selecting for each destination rank s the
    !! Y range [s*ny_l+1, (s+1)*ny_l] it owns after the redistribution.
    !! Shared by the 110 and 010 multi-rank paths.
    implicit none

    real(dp), device, intent(out), dimension(:, :, :, :) :: sendbuf ! (nz_l, nx, ny_l, nproc)
    real(dp), device, intent(in), dimension(:, :, :) :: src ! (nx, ny, nz_l)
    integer, value, intent(in) :: nx, ny_l, nz_l, nproc

    integer :: i, j, k, s

    k = blockIdx%x                                  ! nz_l
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x    ! ny_l

    if (j <= ny_l) then
      do s = 0, nproc - 1
        do i = 1, nx
          sendbuf(k, i, j, s + 1) = src(i, s*ny_l + j, k)
        end do
      end do
    end if

  end subroutine pack_zslab_to_alltoall

  attributes(global) subroutine unpack_110_yslab( &
    dst, recvbuf, nx, ny_l, nz_l, nproc &
    )
    !! Unpacks the MPI_Alltoall receive buffer into the padded real view
    !! (2*(nz/2+1), nx, ny_l) of the cuFFTMp Y-slab descriptor: data
    !! received from source rank s reconstructs the global Z range
    !! [s*nz_l+1, (s+1)*nz_l].
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: dst ! (2*(nz/2+1), nx, ny_l)
    real(dp), device, intent(in), dimension(:, :, :, :) :: recvbuf ! (nz_l, nx, ny_l, nproc)
    integer, value, intent(in) :: nx, ny_l, nz_l, nproc

    integer :: i, j, k, s

    k = blockIdx%x                                  ! nz_l
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x    ! ny_l

    if (j <= ny_l) then
      do s = 0, nproc - 1
        do i = 1, nx
          dst(s*nz_l + k, i, j) = recvbuf(k, i, j, s + 1)
        end do
      end do
    end if

  end subroutine unpack_110_yslab

  attributes(global) subroutine pack_110_yslab( &
    sendbuf, src, nx, ny_l, nz_l, nproc &
    )
    !! Inverse of unpack_110_yslab: packs the Y-slab real view into the
    !! MPI_Alltoall send buffer, selecting for each destination rank s the
    !! Z range it originally owned in the physical Z-slab layout.
    implicit none

    real(dp), device, intent(out), dimension(:, :, :, :) :: sendbuf ! (nz_l, nx, ny_l, nproc)
    real(dp), device, intent(in), dimension(:, :, :) :: src ! (2*(nz/2+1), nx, ny_l)
    integer, value, intent(in) :: nx, ny_l, nz_l, nproc

    integer :: i, j, k, s

    k = blockIdx%x                                  ! nz_l
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x    ! ny_l

    if (j <= ny_l) then
      do s = 0, nproc - 1
        do i = 1, nx
          sendbuf(k, i, j, s + 1) = src(s*nz_l + k, i, j)
        end do
      end do
    end if

  end subroutine pack_110_yslab

  attributes(global) subroutine unpack_alltoall_to_zslab( &
    dst, recvbuf, nx, ny_l, nz_l, nproc &
    )
    !! Inverse of pack_zslab_to_alltoall: unpacks into the physical Z-slab
    !! field block (nx, ny, nz_l). dst must be zeroed by the caller first,
    !! as fft_backward_110_cuda already does for the single rank path.
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: dst ! (nx, ny, nz_l)
    real(dp), device, intent(in), dimension(:, :, :, :) :: recvbuf ! (nz_l, nx, ny_l, nproc)
    integer, value, intent(in) :: nx, ny_l, nz_l, nproc

    integer :: i, j, k, s

    k = blockIdx%x                                  ! nz_l
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x    ! ny_l

    if (j <= ny_l) then
      do s = 0, nproc - 1
        do i = 1, nx
          dst(i, s*ny_l + j, k) = recvbuf(k, i, j, s + 1)
        end do
      end do
    end if

  end subroutine unpack_alltoall_to_zslab

  ! ------------------------------------------------------------------
  ! 010 MULTI-RANK SPECTRAL REDISTRIBUTION
  !
  ! On multiple ranks the stretched y solve of the 010 case needs whole y
  ! lines, but the cuFFTMp spectral slab (nx_spec, ny_l, nz) is split along
  ! Y. The field moves to a Z-slab (nx_spec, ny, nz_l) and back through the
  ! same MPI_Alltoall buffer (nz_l, nx, ny_l, nproc) as the 110 case, with
  ! nx = 2*nx_spec on the real view. The Z-slab side reuses
  ! pack_zslab_to_alltoall and unpack_alltoall_to_zslab. These two kernels
  ! handle the Y-slab side, which has the order (x, y_l, z).
  !
  ! Launch config: the same as the 110 slab redistribution.
  ! ------------------------------------------------------------------

  attributes(global) subroutine pack_010_yslab( &
    sendbuf, src, nx, ny_l, nz_l, nproc &
    )
    !! Packs the real view of the Y-slab (nx, ny_l, nz) into the
    !! MPI_Alltoall send buffer, selecting for each destination rank s the
    !! Z range [s*nz_l+1, (s+1)*nz_l] it owns in the Z-slab layout.
    implicit none

    real(dp), device, intent(out), dimension(:, :, :, :) :: sendbuf ! (nz_l, nx, ny_l, nproc)
    real(dp), device, intent(in), dimension(:, :, :) :: src ! (nx, ny_l, nz)
    integer, value, intent(in) :: nx, ny_l, nz_l, nproc

    integer :: i, j, k, s

    k = blockIdx%x                                  ! nz_l
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x    ! ny_l

    if (j <= ny_l) then
      do s = 0, nproc - 1
        do i = 1, nx
          sendbuf(k, i, j, s + 1) = src(i, j, s*nz_l + k)
        end do
      end do
    end if

  end subroutine pack_010_yslab

  attributes(global) subroutine unpack_010_yslab( &
    dst, recvbuf, nx, ny_l, nz_l, nproc &
    )
    !! Inverse of pack_010_yslab: unpacks the MPI_Alltoall receive buffer
    !! into the real view of the Y-slab (nx, ny_l, nz). Data received from
    !! source rank s fills the global Z range [s*nz_l+1, (s+1)*nz_l].
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: dst ! (nx, ny_l, nz)
    real(dp), device, intent(in), dimension(:, :, :, :) :: recvbuf ! (nz_l, nx, ny_l, nproc)
    integer, value, intent(in) :: nx, ny_l, nz_l, nproc

    integer :: i, j, k, s

    k = blockIdx%x                                  ! nz_l
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x    ! ny_l

    if (j <= ny_l) then
      do s = 0, nproc - 1
        do i = 1, nx
          dst(i, j, s*nz_l + k) = recvbuf(k, i, j, s + 1)
        end do
      end do
    end if

  end subroutine unpack_010_yslab

  attributes(global) subroutine enforce_periodicity_x(f_out, f_in, nx)
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: f_out
    real(dp), device, intent(in), dimension(:, :, :) :: f_in
    integer, value, intent(in) :: nx

    integer :: i, j, k

    j = threadIdx%x
    k = blockIdx%x

    do i = 1, nx
      f_out(i, j, k) = f_in(fold_index(i, nx), j, k)
    end do

  end subroutine enforce_periodicity_x

  attributes(global) subroutine undo_periodicity_x(f_out, f_in, nx)
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: f_out
    real(dp), device, intent(in), dimension(:, :, :) :: f_in
    integer, value, intent(in) :: nx

    integer :: i, j, k

    j = threadIdx%x
    k = blockIdx%x

    do i = 1, nx
      f_out(fold_index(i, nx), j, k) = f_in(i, j, k)
    end do

  end subroutine undo_periodicity_x

  attributes(global) subroutine enforce_periodicity_y(f_out, f_in, ny)
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: f_out
    real(dp), device, intent(in), dimension(:, :, :) :: f_in
    integer, value, intent(in) :: ny

    integer :: i, j, k

    i = threadIdx%x
    k = blockIdx%x

    do j = 1, ny
      f_out(i, j, k) = f_in(i, fold_index(j, ny), k)
    end do

  end subroutine enforce_periodicity_y

  attributes(global) subroutine undo_periodicity_y(f_out, f_in, ny)
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: f_out
    real(dp), device, intent(in), dimension(:, :, :) :: f_in
    integer, value, intent(in) :: ny

    integer :: i, j, k

    i = threadIdx%x
    k = blockIdx%x

    do j = 1, ny
      f_out(i, fold_index(j, ny), k) = f_in(i, j, k)
    end do

  end subroutine undo_periodicity_y

  attributes(global) subroutine enforce_periodicity_xy(f_out, f_in, nx, ny)
    !! Combined X and Y periodicity enforcement (interleave shuffle).
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: f_out
    real(dp), device, intent(in), dimension(:, :, :) :: f_in
    integer, value, intent(in) :: nx, ny

    integer :: i, j, k, src_j

    k = blockIdx%x
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x
    if (j > ny) return

    src_j = fold_index(j, ny)
    do i = 1, nx
      f_out(i, j, k) = f_in(fold_index(i, nx), src_j, k)
    end do

  end subroutine enforce_periodicity_xy

  attributes(global) subroutine undo_periodicity_xy(f_out, f_in, nx, ny)
    !! Combined X and Y periodicity undo (reverse interleave shuffle).
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: f_out
    real(dp), device, intent(in), dimension(:, :, :) :: f_in
    integer, value, intent(in) :: nx, ny

    integer :: i, j, k, dst_j

    k = blockIdx%x
    j = (blockIdx%y - 1)*blockDim%x + threadIdx%x
    if (j > ny) return

    dst_j = fold_index(j, ny)
    do i = 1, nx
      f_out(fold_index(i, nx), dst_j, k) = f_in(i, j, k)
    end do

  end subroutine undo_periodicity_xy

  attributes(device) integer function fold_index(i, n) result(src)
    !! Source index of entry i of a non-periodic line of n points gathered
    !! into a periodic one: the odd entries ascending, then the even
    !! entries descending, so that a plain FFT can stand in for the cosine
    !! transform. For an odd n the last entry lands at the centre, n/2 + 1.
    !! The single definition of the fold for the periodicity kernels: the
    !! enforce kernels gather through it and the undo kernels scatter
    !! through it, which keeps their reads contiguous. The OpenMP backend
    !! has the same function.
    implicit none

    integer, value, intent(in) :: i, n

    if (i <= n/2) then
      src = 2*i - 1
    else if (mod(n, 2) == 1 .and. i == n/2 + 1) then
      src = n
    else
      src = 2*n - 2*i + 2
    end if

  end function fold_index

end module m_cuda_spectral