fieldops.f90 Source File


This file depends on

sourcefile~~fieldops.f90~~EfferentGraph sourcefile~fieldops.f90 fieldops.f90 sourcefile~common.f90~3 common.f90 sourcefile~fieldops.f90->sourcefile~common.f90~3 sourcefile~common.f90~4 common.f90 sourcefile~fieldops.f90->sourcefile~common.f90~4 sourcefile~logger.f90 logger.f90 sourcefile~common.f90~3->sourcefile~logger.f90

Files dependent on this one

sourcefile~~fieldops.f90~~AfferentGraph sourcefile~fieldops.f90 fieldops.f90 sourcefile~backend.f90~4 backend.f90 sourcefile~backend.f90~4->sourcefile~fieldops.f90 sourcefile~setup.f90 setup.f90 sourcefile~setup.f90->sourcefile~backend.f90~4 sourcefile~build.f90 build.f90 sourcefile~build.f90->sourcefile~setup.f90 sourcefile~poisson_solver.f90 poisson_solver.f90 sourcefile~poisson_solver.f90->sourcefile~setup.f90 sourcefile~xcompact.f90 xcompact.f90 sourcefile~xcompact.f90->sourcefile~setup.f90 sourcefile~memcheck.f90 memcheck.f90 sourcefile~memcheck.f90->sourcefile~build.f90

Source Code

module m_cuda_kernels_fieldops
  use cudafor

  use m_common, only: dp
  use m_cuda_common, only: SZ

contains

  attributes(global) subroutine copy(n, dst, src)
    implicit none

    integer, value, intent(in) :: n
    real(dp), device, intent(out), dimension(:, :, :) :: dst
    real(dp), device, intent(in), dimension(:, :, :) :: src

    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      dst(i, j, b) = src(i, j, b)
    end do

  end subroutine copy

  attributes(global) subroutine axpby(n, alpha, x, beta, y)
    implicit none

    integer, value, intent(in) :: n
    real(dp), value, intent(in) :: alpha, beta
    real(dp), device, intent(in), dimension(:, :, :) :: x
    real(dp), device, intent(inout), dimension(:, :, :) :: y

    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      y(i, j, b) = alpha*x(i, j, b) + beta*y(i, j, b)
    end do

  end subroutine axpby

  attributes(global) subroutine pwmul(y, x, n)
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: y
    real(dp), device, intent(in), dimension(:, :, :) :: x
    integer, value, intent(in) :: n

    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      y(i, j, b) = y(i, j, b)*x(i, j, b)
    end do

  end subroutine pwmul

  attributes(global) subroutine buffer_copy(u_send_s, u_send_e, u, n, n_halo)
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: u_send_s, u_send_e
    real(dp), device, intent(in), dimension(:, :, :) :: u
    integer, value, intent(in) :: n, n_halo

    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n_halo
      u_send_s(i, j, b) = u(i, j, b)
      u_send_e(i, j, b) = u(i, n - n_halo + j, b)
    end do

  end subroutine buffer_copy

  attributes(global) subroutine field_scale(f, alpha, n)
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), value, intent(in) :: alpha
    integer, value, intent(in) :: n

    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      f(i, j, b) = alpha*f(i, j, b)
    end do

  end subroutine field_scale

  attributes(global) subroutine field_shift(f, const, n)
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), value, intent(in) :: const
    integer, value, intent(in) :: n

    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      f(i, j, b) = f(i, j, b) + const
    end do

  end subroutine field_shift

  attributes(global) subroutine vorticity_from_gradients( &
    vort, dudy, dudz, dvdx, dvdz, dwdx, dwdy, n)
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: vort
    real(dp), device, intent(in), dimension(:, :, :) :: &
      dudy, dudz, dvdx, dvdz, dwdx, dwdy
    integer, value, intent(in) :: n

    integer :: i, j, b
    real(dp) :: wx, wy, wz

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      wx = dwdy(i, j, b) - dvdz(i, j, b)
      wy = dudz(i, j, b) - dwdx(i, j, b)
      wz = dvdx(i, j, b) - dudy(i, j, b)
      vort(i, j, b) = sqrt(wx*wx + wy*wy + wz*wz)
    end do

  end subroutine vorticity_from_gradients

  attributes(global) subroutine qcriterion_from_gradients( &
    qcrit, dudx, dudy, dudz, dvdx, dvdy, dvdz, dwdx, dwdy, dwdz, n)
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: qcrit
    real(dp), device, intent(in), dimension(:, :, :) :: dudx, dudy, dudz
    real(dp), device, intent(in), dimension(:, :, :) :: dvdx, dvdy, dvdz
    real(dp), device, intent(in), dimension(:, :, :) :: dwdx, dwdy, dwdz
    integer, value, intent(in) :: n

    integer :: i, j, b
    real(dp) :: divu

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      divu = dudx(i, j, b) + dvdy(i, j, b) + dwdz(i, j, b)
      qcrit(i, j, b) = 0.5_dp*divu*divu - &
                       0.5_dp*(dudx(i, j, b)*dudx(i, j, b) + &
                               dvdy(i, j, b)*dvdy(i, j, b) + &
                               dwdz(i, j, b)*dwdz(i, j, b)) - &
                       dudy(i, j, b)*dvdx(i, j, b) - &
                       dudz(i, j, b)*dwdx(i, j, b) - &
                       dvdz(i, j, b)*dwdy(i, j, b)
    end do

  end subroutine qcriterion_from_gradients

  attributes(global) subroutine smagorinsky_from_gradients( &
    nut, mixing_length_sq, dudx, dudy, dudz, dvdx, dvdy, dvdz, &
    dwdx, dwdy, dwdz, n)
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: nut
    real(dp), device, intent(in), dimension(:, :, :) :: mixing_length_sq
    real(dp), device, intent(in), dimension(:, :, :) :: dudx, dudy, dudz
    real(dp), device, intent(in), dimension(:, :, :) :: dvdx, dvdy, dvdz
    real(dp), device, intent(in), dimension(:, :, :) :: dwdx, dwdy, dwdz
    integer, value, intent(in) :: n

    real(dp) :: sij_sq
    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      if (mixing_length_sq(i, j, b) > 0._dp) then
        sij_sq = dudx(i, j, b)**2 + dvdy(i, j, b)**2 + &
                 dwdz(i, j, b)**2 + &
                 0.5_dp*(dudy(i, j, b) + dvdx(i, j, b))**2 + &
                 0.5_dp*(dudz(i, j, b) + dwdx(i, j, b))**2 + &
                 0.5_dp*(dvdz(i, j, b) + dwdy(i, j, b))**2
        nut(i, j, b) = mixing_length_sq(i, j, b)*sqrt(2._dp*sij_sq)
      else
        nut(i, j, b) = 0._dp
      end if
    end do

  end subroutine smagorinsky_from_gradients

  attributes(global) subroutine sgs_stress_from_gradients( &
    stress, nut, gradient_a, gradient_b, scale_a, scale_b, n)
    implicit none

    real(dp), device, intent(out), dimension(:, :, :) :: stress
    real(dp), device, intent(in), dimension(:, :, :) :: &
      nut, gradient_a, gradient_b
    real(dp), value, intent(in) :: scale_a, scale_b
    integer, value, intent(in) :: n

    integer :: i, j, b

    i = threadIdx%x
    b = blockIdx%x

    do j = 1, n
      if (nut(i, j, b) > 0._dp) then
        stress(i, j, b) = nut(i, j, b)* &
                          (scale_a*gradient_a(i, j, b) &
                           + scale_b*gradient_b(i, j, b))
      else
        stress(i, j, b) = 0._dp
      end if
    end do
  end subroutine sgs_stress_from_gradients

  attributes(global) subroutine scalar_product(s, x, y, n, n_i_pad, n_j)
    implicit none

    real(dp), device, intent(inout) :: s
    real(dp), device, intent(in), dimension(:, :, :) :: x, y
    integer, value, intent(in) :: n, n_i_pad, n_j

    real(dp) :: s_pncl !! pencil sum
    integer :: i, j, b, b_i, b_j, ierr

    i = threadIdx%x
    b_i = blockIdx%x
    b_j = blockIdx%y

    b = b_i + (b_j - 1)*n_i_pad
    s_pncl = 0._dp
    if (i + (b_j - 1)*blockDim%x <= n_j) then
      do j = 1, n
        s_pncl = s_pncl + x(i, j, b)*y(i, j, b)
      end do
    end if
    ierr = atomicadd(s, s_pncl)

  end subroutine scalar_product

  attributes(global) subroutine vector_norm_squared(s, a, b, c, &
                                                    n, n_i_pad, n_j)
    implicit none

    real(dp), device, intent(inout) :: s
    real(dp), device, intent(in), dimension(:, :, :) :: a, b, c
    integer, value, intent(in) :: n, n_i_pad, n_j

    real(dp) :: pencil_sum
    integer :: i, j, pencil, block_i, block_j, ierr

    i = threadIdx%x
    block_i = blockIdx%x
    block_j = blockIdx%y
    pencil = block_i + (block_j - 1)*n_i_pad

    pencil_sum = 0._dp
    if (i + (block_j - 1)*blockDim%x <= n_j) then
      do j = 1, n
        pencil_sum = pencil_sum + a(i, j, pencil)**2 + &
                     b(i, j, pencil)**2 + c(i, j, pencil)**2
      end do
    end if
    ierr = atomicadd(s, pencil_sum)
  end subroutine vector_norm_squared

  attributes(global) subroutine labelled_sums(s, g, label, a, b, c, &
                                              n_label, n, n_i_pad, n_j)
    !! s(1:3, l) += sums of g*a, g*b and g*c over the points labelled l;
    !! launch as vector_norm_squared. Each thread accumulates runs of equal
    !! labels along its pencil and flushes a run when the label changes, so
    !! the atomics scale with the label changes rather than the points.
    implicit none

    real(dp), device, intent(inout) :: s(3, n_label)
    real(dp), device, intent(in), dimension(:, :, :) :: g, label, a, b, c
    integer, value, intent(in) :: n_label, n, n_i_pad, n_j

    real(dp) :: sum_a, sum_b, sum_c
    integer :: i, j, l, l_run, pencil, block_i, block_j, ierr

    i = threadIdx%x
    block_i = blockIdx%x
    block_j = blockIdx%y
    pencil = block_i + (block_j - 1)*n_i_pad

    if (i + (block_j - 1)*blockDim%x > n_j) return

    l_run = 0
    sum_a = 0._dp
    sum_b = 0._dp
    sum_c = 0._dp
    do j = 1, n
      l = nint(label(i, j, pencil))
      if (l < 1 .or. l > n_label) l = 0
      if (l /= l_run) then
        if (l_run /= 0) then
          ierr = atomicadd(s(1, l_run), sum_a)
          ierr = atomicadd(s(2, l_run), sum_b)
          ierr = atomicadd(s(3, l_run), sum_c)
        end if
        l_run = l
        sum_a = 0._dp
        sum_b = 0._dp
        sum_c = 0._dp
      end if
      if (l /= 0) then
        sum_a = sum_a + g(i, j, pencil)*a(i, j, pencil)
        sum_b = sum_b + g(i, j, pencil)*b(i, j, pencil)
        sum_c = sum_c + g(i, j, pencil)*c(i, j, pencil)
      end if
    end do
    if (l_run /= 0) then
      ierr = atomicadd(s(1, l_run), sum_a)
      ierr = atomicadd(s(2, l_run), sum_b)
      ierr = atomicadd(s(3, l_run), sum_c)
    end if
  end subroutine labelled_sums

  attributes(global) subroutine labelled_axpy(coeffs, g, label, a, b, c, &
                                              n_label, n, n_i_pad, n_j)
    !! a, b, c += coeffs(1:3, l)*g at the points labelled l; launch as
    !! vector_norm_squared.
    implicit none

    real(dp), device, intent(in) :: coeffs(3, n_label)
    real(dp), device, intent(in), dimension(:, :, :) :: g, label
    real(dp), device, intent(inout), dimension(:, :, :) :: a, b, c
    integer, value, intent(in) :: n_label, n, n_i_pad, n_j

    integer :: i, j, l, pencil, block_i, block_j

    i = threadIdx%x
    block_i = blockIdx%x
    block_j = blockIdx%y
    pencil = block_i + (block_j - 1)*n_i_pad

    if (i + (block_j - 1)*blockDim%x > n_j) return

    do j = 1, n
      l = nint(label(i, j, pencil))
      if (l < 1 .or. l > n_label) cycle
      a(i, j, pencil) = a(i, j, pencil) + coeffs(1, l)*g(i, j, pencil)
      b(i, j, pencil) = b(i, j, pencil) + coeffs(2, l)*g(i, j, pencil)
      c(i, j, pencil) = c(i, j, pencil) + coeffs(3, l)*g(i, j, pencil)
    end do
  end subroutine labelled_axpy

  attributes(global) subroutine field_max_sum(max_f, sum_f, f, n, n_i_pad, n_j)
    implicit none

    real(dp), device, intent(inout) :: max_f, sum_f
    real(dp), device, intent(in), dimension(:, :, :) :: f
    integer, value, intent(in) :: n, n_i_pad, n_j

    real(dp) :: max_pncl, sum_pncl, val
    integer :: i, j, b, b_i, b_j, ierr

    i = threadIdx%x
    b_i = blockIdx%x
    b_j = blockIdx%y

    b = b_i + (b_j - 1)*n_i_pad
    max_pncl = 0._dp
    sum_pncl = 0._dp
    if (i + (b_j - 1)*blockDim%x <= n_j) then
      do j = 1, n
        val = abs(f(i, j, b))
        sum_pncl = sum_pncl + val
        max_pncl = max(max_pncl, val)
      end do
    end if
    ierr = atomicadd(sum_f, sum_pncl)
    ierr = atomicmax(max_f, max_pncl)

  end subroutine field_max_sum

  attributes(global) subroutine field_set_y_face( &
    f, c_start, c_end, set_start, set_end, flow_rate_diff, nx, ny, nz)
  !! Set domain Y_FACE boundary values.
  !! c_start: Dirichlet value applied at the bottom face (j = 1)
  !! c_end:   Dirichlet value applied at the top face (j = ny)
  !! set_start/set_end: whether that face carries a prescribed value at all,
  !! so the two y-faces can differ (ABL: no-slip floor, free-slip lid).
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), value, intent(in) :: c_start, c_end, flow_rate_diff
    logical, value, intent(in) :: set_start, set_end
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, b, n_mod, b_end

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x ! from 1 to nx
    b = blockIdx%y ! from 1 to nz

    ! DIR_X groups are ordered z fastest: b = (y_block - 1)*nz + z, so the
    ! bottom face lives in groups 1..nz and the top face nz*(n_y_blocks-1)+z.
    n_mod = mod(ny - 1, SZ) + 1
    b_end = b + (ny - 1)/SZ*nz

    if (j <= nx) then
      if (set_start) f(1, j, b) = c_start
      if (set_end) f(n_mod, j, b_end) = c_end
    end if

  end subroutine field_set_y_face
  attributes(global) subroutine field_plane_partial_sums( &
    partial, f, nx, ny, nz)
  !! First pass of field_plane_sums: partial(j, z) is the sum over x of
  !! y index j at z. Launch with threads = dim3(SZ, 1, 1) and
  !! blocks = dim3(n_y_blocks, nz, 1).
    implicit none

    real(dp), device, intent(out), dimension(:, :) :: partial
    real(dp), device, intent(in), dimension(:, :, :) :: f
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, j, k, b
    real(dp) :: line_sum

    i = threadIdx%x
    j = i + (blockIdx%x - 1)*SZ
    if (j > ny) return

    ! DIR_X groups are ordered z fastest: b = (y_block - 1)*nz + z
    b = (blockIdx%x - 1)*nz + blockIdx%y
    line_sum = 0._dp
    do k = 1, nx
      line_sum = line_sum + f(i, k, b)
    end do
    partial(j, blockIdx%y) = line_sum

  end subroutine field_plane_partial_sums

  attributes(global) subroutine field_plane_sums_reduce(sums, partial, ny, nz)
  !! Second pass of field_plane_sums: sums(j) = sum over z of partial(j, z),
  !! in a fixed order. Launch with one thread per y index.
    implicit none

    real(dp), device, intent(out), dimension(:) :: sums
    real(dp), device, intent(in), dimension(:, :) :: partial
    integer, value, intent(in) :: ny, nz

    integer :: j, z
    real(dp) :: plane_sum

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    if (j > ny) return

    plane_sum = 0._dp
    do z = 1, nz
      plane_sum = plane_sum + partial(j, z)
    end do
    sums(j) = plane_sum

  end subroutine field_plane_sums_reduce

  attributes(global) subroutine field_set_y_plane(f, c, i_in_block, &
                                                  group_offset, nx)
  !! Set one interior y-plane to a constant. The caller resolves the plane
  !! into its position within a y-block and the group offset for that block,
  !! so this kernel stays free of layout arithmetic.
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), value, intent(in) :: c
    integer, value, intent(in) :: i_in_block, group_offset, nx

    integer :: j, b

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x ! from 1 to nx
    b = group_offset + blockIdx%y ! z fastest within the y-block

    if (j <= nx) f(i_in_block, j, b) = c

  end subroutine field_set_y_plane

  attributes(global) subroutine field_set_abl_wall_stress( &
    stress, u, w, drag_coeff, upper_weight, sample_i, upper_i, stress_i, &
    sample_offset, upper_offset, stress_offset, nx, component)
  !! Drag law tau = drag_coeff*u_c*|u_h| from each column's own sampled
  !! velocity, written into one stress plane (component 1: x, 3: z). The
  !! velocity is interpolated between the sample plane and the one above it.
    implicit none
    real(dp), device, intent(inout), dimension(:, :, :) :: stress
    real(dp), device, intent(in), dimension(:, :, :) :: u, w
    real(dp), value, intent(in) :: drag_coeff, upper_weight
    integer, value, intent(in) :: sample_i, upper_i, stress_i
    integer, value, intent(in) :: sample_offset, upper_offset
    integer, value, intent(in) :: stress_offset, nx, component
    integer :: i, sample_group, upper_group, stress_group
    real(dp) :: us, ws, speed

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    sample_group = sample_offset + blockIdx%y
    upper_group = upper_offset + blockIdx%y
    stress_group = stress_offset + blockIdx%y
    if (i > nx) return
    us = (1._dp - upper_weight)*u(sample_i, i, sample_group) &
         + upper_weight*u(upper_i, i, upper_group)
    ws = (1._dp - upper_weight)*w(sample_i, i, sample_group) &
         + upper_weight*w(upper_i, i, upper_group)
    speed = sqrt(us**2 + ws**2)
    if (component == 1) then
      stress(stress_i, i, stress_group) = drag_coeff*us*speed
    else
      stress(stress_i, i, stress_group) = drag_coeff*ws*speed
    end if
  end subroutine field_set_abl_wall_stress

  attributes(global) subroutine field_get_x_band(band, f, i_start, nz)
  !! band(i, j, k) = f(i_start + i - 1, j, k) for a DIR_X vertex field;
  !! one thread per i, one block per (j, k)
    implicit none
    real(dp), device, intent(out), dimension(:, :, :) :: band
    real(dp), device, intent(in), dimension(:, :, :) :: f
    integer, value, intent(in) :: i_start, nz
    integer :: i, j, k

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    j = blockIdx%y
    k = blockIdx%z
    if (i > size(band, 1)) return
    band(i, j, k) = f(mod(j - 1, SZ) + 1, i_start + i - 1, &
                      ((j - 1)/SZ)*nz + k)
  end subroutine field_get_x_band

  attributes(global) subroutine field_blend_x_band(f, band, lambda, &
                                                   i_start, nz)
  !! f(i_start + i - 1, j, k) = lambda(i)*band(i, j, k)
  !!                            + (1 - lambda(i))*f(i_start + i - 1, j, k)
  !! for a DIR_X vertex field; one thread per i, one block per (j, k)
    implicit none
    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), device, intent(in), dimension(:, :, :) :: band
    real(dp), device, intent(in), dimension(:) :: lambda
    integer, value, intent(in) :: i_start, nz
    integer :: i, j, k, jj, g

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    j = blockIdx%y
    k = blockIdx%z
    if (i > size(band, 1)) return
    jj = mod(j - 1, SZ) + 1
    g = ((j - 1)/SZ)*nz + k
    f(jj, i_start + i - 1, g) = lambda(i)*band(i, j, k) &
                                + (1._dp - lambda(i)) &
                                *f(jj, i_start + i - 1, g)
  end subroutine field_blend_x_band

  attributes(global) subroutine field_set_y_face_from_field( &
    f, f_start, nx, ny, nz)
!! Set domain Y_FACE boundary values from another field.
!! Both f and f_start are DIR_X VERT pencil-layout. Only the bottom
!! (j = 1) and top (j = ny) y-pencil planes of f_start are read.
!!
!! Launch convention matches field_set_y_face exactly:
!!   threads = dim3(64, 1, 1)
!!   blocks  = dim3((nx-1)/64 + 1, nz, 1)
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), device, intent(in), dimension(:, :, :) :: f_start
    integer, value, intent(in) :: nx, ny, nz

    integer :: j, b, n_mod, b_end

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x ! from 1 to nx
    b = blockIdx%y ! from 1 to nz

    n_mod = mod(ny - 1, SZ) + 1
    b_end = b + (ny - 1)/SZ*nz

    if (j <= nx) then
      f(1, j, b) = f_start(1, j, b)
      f(n_mod, j, b_end) = f_start(n_mod, j, b_end)
    end if

  end subroutine field_set_y_face_from_field
  attributes(global) subroutine field_set_x_face( &
    f, c_start, c_end, bc_start, bc_end, flow_rate_diff, nx, ny, nz)
  !! Set domain X_FACE boundary values.
  !! c_start: Dirichlet value applied at the left face (i = 1)
  !! c_end:   convective velocity Uc = uxmax * gdt / dx,
  !!          used as multiplier in the outflow scheme du/dt + Uc*du/dx = 0
    implicit none
    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), value, intent(in) :: c_start, c_end, flow_rate_diff
    integer, value, intent(in) :: bc_start, bc_end
    integer, value, intent(in) :: nx, ny, nz
    integer :: i, b, n_mod, n_y_blocks, y_block, i_max

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! 1..SZ
    b = blockIdx%y                                 ! 1..n_y_blocks*nz

    n_mod = mod(ny - 1, SZ) + 1
    n_y_blocks = (ny - 1)/SZ + 1

    ! Determine if this b is the last y-block (padding present)
    ! With b = (y_block - 1)*nz + z, y_block = (b - 1)/nz + 1
    y_block = (b - 1)/nz + 1
    if (y_block == n_y_blocks) then
      i_max = n_mod    ! last y-block: only i in [1, n_mod] are real
    else
      i_max = SZ       ! interior y-blocks: all i in [1, SZ] are real
    end if

    if (i <= i_max) then
      ! --- Left face (j = 1) ---
      select case (bc_start)
      case (1) ! BC_NEUMANN
        !this can be empty for now future TODO
      case (2) ! BC_DIRICHLET
        f(i, 1, b) = c_start
      end select

      ! --- Right face (j = nx) ---
      select case (bc_end)
      case (1) !BC_NEUMANN
        !this can be empty for now future TODO
      case (2) ! BC_DIRICHLET
        f(i, nx, b) = f(i, nx, b) &
                      - c_end*(f(i, nx, b) - f(i, nx - 1, b)) &
                      + flow_rate_diff
      end select
    end if
  end subroutine field_set_x_face

  attributes(global) subroutine field_set_x_face_from_field( &
    f, f_start, c_end, bc_start, bc_end, flow_rate_diff, nx, ny, nz)
  !! Set domain X_FACE boundary values with a spatially-varying inlet.
  !!
  !! Both `f` and `f_start` are DIR_X VERT fields with shape
  !! (SZ, nx, n_y_blocks*nz). The inlet plane is pencil index 1 of each,
  !! so f(i, 1, b) <- f_start(i, 1, b) for every real (i, b). Only the
  !! i = 1 pencil-index plane of f_start is read; the rest is ignored.
  !!
  !! Launch convention (matches field_set_x_face exactly):
  !!   threads = dim3(64, 1, 1)
  !!   blocks  = dim3((SZ-1)/64 + 1, n_y_blocks*nz, 1)
  !! Threads beyond the real pencil width (SZ in interior y-blocks,
  !! n_mod in the last y-block) are masked out by the i_max guard.
  !!
  !! c_end is the convective velocity Uc = uxmax * gdt / dx used in the
  !! outflow scheme du/dt + Uc*du/dx = 0 at the right face.
    implicit none
    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), device, intent(in), dimension(:, :, :) :: f_start
    real(dp), value, intent(in) :: c_end, flow_rate_diff
    integer, value, intent(in) :: bc_start, bc_end
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, b, n_mod, n_y_blocks, y_block, i_max

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! 1..SZ (or n_mod)
    b = blockIdx%y                                 ! 1..n_y_blocks*nz

    n_mod = mod(ny - 1, SZ) + 1
    n_y_blocks = (ny - 1)/SZ + 1
    y_block = (b - 1)/nz + 1

    if (y_block == n_y_blocks) then
      i_max = n_mod    ! last y-block: only i in [1, n_mod] are real
    else
      i_max = SZ       ! interior y-blocks: all i in [1, SZ] are real
    end if

    if (i <= i_max) then
      ! --- Left face (pencil index 1, i.e. global x = 1) ---
      select case (bc_start)
      case (1) ! BC_NEUMANN
        ! future TODO
      case (2) ! BC_DIRICHLET
        f(i, 1, b) = f_start(i, 1, b)
      end select

      ! --- Right face (pencil index nx, i.e. global x = nx) ---
      select case (bc_end)
      case (1) ! BC_NEUMANN
        ! future TODO
      case (2) ! BC_DIRICHLET
        f(i, nx, b) = f(i, nx, b) &
                      - c_end*(f(i, nx, b) - f(i, nx - 1, b)) &
                      + flow_rate_diff
      end select
    end if
  end subroutine field_set_x_face_from_field

  attributes(global) subroutine field_set_y_face_from_plane( &
    f, p_start, p_end, apply_end, nx, ny, nz)
    !! Write two (nx, nz) planes into the y = 1 and y = ny planes of a
    !! DIR_X field. p_end is read only when apply_end = 1.
    !! Launch convention matches field_set_y_face_from_field.
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), device, intent(in), dimension(:, :) :: p_start, p_end
    integer, value, intent(in) :: apply_end, nx, ny, nz

    integer :: j, b, n_mod, b_end

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x ! from 1 to nx
    b = blockIdx%y ! from 1 to nz

    n_mod = mod(ny - 1, SZ) + 1
    b_end = b + (ny - 1)/SZ*nz

    if (j <= nx) then
      f(1, j, b) = p_start(j, b)
      if (apply_end == 1) f(n_mod, j, b_end) = p_end(j, b)
    end if

  end subroutine field_set_y_face_from_plane

  attributes(global) subroutine field_set_x_face_from_plane( &
    f, p_start, p_end, apply_end, nx, ny, nz)
    !! Write two (ny, nz) planes into the x = 1 and x = nx planes of a
    !! DIR_X field. p_end is read only when apply_end = 1.
    !! Launch convention matches field_set_x_face_from_field.
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), device, intent(in), dimension(:, :) :: p_start, p_end
    integer, value, intent(in) :: apply_end, nx, ny, nz

    integer :: i, b, n_mod, n_y_blocks, y_block, i_max, y, z

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! 1..SZ
    b = blockIdx%y                                 ! 1..n_y_blocks*nz

    n_mod = mod(ny - 1, SZ) + 1
    n_y_blocks = (ny - 1)/SZ + 1
    ! CUDA DIR_X ordering: b = (y_block - 1)*nz + z
    y_block = (b - 1)/nz + 1
    z = b - (y_block - 1)*nz
    y = (y_block - 1)*SZ + i

    if (y_block == n_y_blocks) then
      i_max = n_mod
    else
      i_max = SZ
    end if

    if (i <= i_max) then
      f(i, 1, b) = p_start(y, z)
      if (apply_end == 1) f(i, nx, b) = p_end(y, z)
    end if

  end subroutine field_set_x_face_from_plane

  attributes(global) subroutine add_scalar_x_face( &
    f, c, i_plane, nx, ny, nz)
  !! Adds a uniform constant `c` to a single x-normal plane (pencil index
  !! i_plane) of a DIR_X VERT field, in place.
  !!
  !! Launch convention (matches field_set_x_face exactly):
  !!   threads = dim3(64, 1, 1)
  !!   blocks  = dim3((SZ-1)/64 + 1, n_y_blocks*nz, 1)
  !! Threads beyond the real pencil width (SZ in interior y-blocks,
  !! n_mod in the last y-block) are masked out by the i_max guard.
    implicit none
    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), value, intent(in) :: c
    integer, value, intent(in) :: i_plane, nx, ny, nz

    integer :: i, b, n_mod, n_y_blocks, y_block, i_max

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x  ! 1..SZ (or n_mod)
    b = blockIdx%y                                 ! 1..n_y_blocks*nz

    n_mod = mod(ny - 1, SZ) + 1
    n_y_blocks = (ny - 1)/SZ + 1
    y_block = (b - 1)/nz + 1

    if (y_block == n_y_blocks) then
      i_max = n_mod    ! last y-block: only i in [1, n_mod] are real
    else
      i_max = SZ       ! interior y-blocks: all i in [1, SZ] are real
    end if

    if (i <= i_max) then
      f(i, i_plane, b) = f(i, i_plane, b) + c
    end if
  end subroutine add_scalar_x_face

  attributes(global) subroutine field_add_x_face_from_field( &
    f, g, set_start, set_end, nx, ny, nz)
  !! f += g on the x-faces (pencil index 1 and nx) flagged by
  !! set_start/set_end. Launch as field_set_x_face_from_field.
    implicit none
    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), device, intent(in), dimension(:, :, :) :: g
    logical, value, intent(in) :: set_start, set_end
    integer, value, intent(in) :: nx, ny, nz

    integer :: i, b, n_mod, i_max

    i = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    b = blockIdx%y

    ! Groups are z fast, so the last y-block is b > nz*(n_y_blocks - 1)
    n_mod = mod(ny - 1, SZ) + 1
    if ((b - 1)/nz + 1 == (ny - 1)/SZ + 1) then
      i_max = n_mod
    else
      i_max = SZ
    end if

    if (i <= i_max) then
      if (set_start) f(i, 1, b) = f(i, 1, b) + g(i, 1, b)
      if (set_end) f(i, nx, b) = f(i, nx, b) + g(i, nx, b)
    end if
  end subroutine field_add_x_face_from_field

  attributes(global) subroutine field_add_y_face_from_field( &
    f, g, set_start, set_end, nx, ny, nz)
  !! f += g on the y-faces (j = 1 and ny) flagged by set_start/set_end.
  !! Launch as field_set_y_face.
    implicit none
    real(dp), device, intent(inout), dimension(:, :, :) :: f
    real(dp), device, intent(in), dimension(:, :, :) :: g
    logical, value, intent(in) :: set_start, set_end
    integer, value, intent(in) :: nx, ny, nz

    integer :: j, b, n_mod, b_end

    j = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    b = blockIdx%y

    n_mod = mod(ny - 1, SZ) + 1
    b_end = b + (ny - 1)/SZ*nz

    if (j <= nx) then
      if (set_start) f(1, j, b) = f(1, j, b) + g(1, j, b)
      if (set_end) f(n_mod, j, b_end) = f(n_mod, j, b_end) + g(n_mod, j, b_end)
    end if
  end subroutine field_add_y_face_from_field

  attributes(global) subroutine volume_integral(s, f, n, n_i_pad, n_j)
    implicit none

    real(dp), device, intent(inout) :: s
    real(dp), device, intent(in), dimension(:, :, :) :: f
    integer, value, intent(in) :: n, n_i_pad, n_j

    real(dp) :: s_pncl !! pencil sum
    integer :: i, j, b, b_i, b_j, ierr

    i = threadIdx%x
    b_i = blockIdx%x
    b_j = blockIdx%y

    b = b_i + (b_j - 1)*n_i_pad
    s_pncl = 0._dp
    if (i + (b_j - 1)*blockDim%x <= n_j) then
      do j = 1, n
        s_pncl = s_pncl + f(i, j, b)
      end do
    end if
    ierr = atomicadd(s, s_pncl)

  end subroutine volume_integral

  attributes(global) subroutine sample_points( &
    vals, points, a, b, c, n_points, dx, dy, dz, nx, ny, nz, gx, gy, gz, &
    ox, oy, oz, px, py, pz, n_i_pad &
    )
    !! vals(1:3, p) = this rank's part of the trilinear interpolation of a, b
    !! and c at points(:, p), one thread per point; see field_sample_points.
    !! (dx, dy, dz) is the grid spacing, (nx, ny, nz) and (gx, gy, gz) the
    !! local and global VERT dims, (ox, oy, oz) the rank offsets and
    !! (px, py, pz) flag periodic directions.
    implicit none

    real(dp), device, intent(inout) :: vals(3, n_points)
    real(dp), device, intent(in) :: points(3, n_points)
    real(dp), device, intent(in), dimension(:, :, :) :: a, b, c
    integer, value, intent(in) :: n_points, nx, ny, nz, gx, gy, gz, &
                                  ox, oy, oz, n_i_pad
    real(dp), value, intent(in) :: dx, dy, dz
    logical, value, intent(in) :: px, py, pz

    real(dp) :: fx, fy, fz, w, sa, sb, sc
    integer :: p, o1, o2, o3, x0, y0, z0, lx, ly, lz, i, pencil

    p = threadIdx%x + (blockIdx%x - 1)*blockDim%x
    if (p > n_points) return

    fx = points(1, p)/dx
    fy = points(2, p)/dy
    fz = points(3, p)/dz
    x0 = floor(fx)
    y0 = floor(fy)
    z0 = floor(fz)
    fx = fx - x0
    fy = fy - y0
    fz = fz - z0

    sa = 0._dp
    sb = 0._dp
    sc = 0._dp
    do o3 = 0, 1
      lz = local_index(z0 + o3, gz, oz, nz, pz)
      if (lz < 1) cycle
      do o2 = 0, 1
        ly = local_index(y0 + o2, gy, oy, ny, py)
        if (ly < 1) cycle
        do o1 = 0, 1
          lx = local_index(x0 + o1, gx, ox, nx, px)
          if (lx < 1) cycle
          w = merge(fx, 1._dp - fx, o1 == 1)*merge(fy, 1._dp - fy, o2 == 1) &
              *merge(fz, 1._dp - fz, o3 == 1)
          ! DIR_X storage: x along j, y split into SZ-blocks, z fastest
          i = mod(ly - 1, SZ) + 1
          pencil = lz + ((ly - 1)/SZ)*n_i_pad
          sa = sa + w*a(i, lx, pencil)
          sb = sb + w*b(i, lx, pencil)
          sc = sc + w*c(i, lx, pencil)
        end do
      end do
    end do
    vals(1, p) = sa
    vals(2, p) = sb
    vals(3, p) = sc
  end subroutine sample_points

  attributes(device) integer function local_index(g, n_glob, offset, &
                                                  n_loc, periodic)
    !! Local one-based index of the zero-based global index g, wrapped when
    !! periodic; 0 when it is off this rank or outside the domain.
    implicit none

    integer, value, intent(in) :: g, n_glob, offset, n_loc
    logical, value, intent(in) :: periodic

    integer :: gw

    gw = g
    if (periodic) gw = modulo(g, n_glob)
    local_index = gw - offset + 1
    if (gw < 0 .or. gw >= n_glob .or. local_index < 1 &
        .or. local_index > n_loc) local_index = 0
  end function local_index

  attributes(global) subroutine add_gaussian_sources( &
    a, b, c, src, box, n_src, inv_eps2, r_cut2, dx, dy, dz, nx, ny, nz, &
    gx, gy, gz, ox, oy, oz, px, py, pz, n_i_pad &
    )
    !! a, b, c += src(4:6, s)*exp(-dist^2/eps^2) within r_cut of the point
    !! src(1:3, s), over the global index box box(1:3, s)..box(4:6, s); see
    !! source_index_boxes. Box indices are wrapped across periodic
    !! boundaries and skipped where this rank does not own them. Launch
    !! with blocks (n_src, box extent in z) and threads striding y within
    !! the box; sources overlap, so the adds are atomic.
    implicit none

    real(dp), device, intent(inout), dimension(:, :, :) :: a, b, c
    real(dp), device, intent(in) :: src(6, n_src)
    integer, device, intent(in) :: box(6, n_src)
    integer, value, intent(in) :: n_src, nx, ny, nz, gx, gy, gz, &
                                  ox, oy, oz, n_i_pad
    real(dp), value, intent(in) :: inv_eps2, r_cut2, dx, dy, dz
    logical, value, intent(in) :: px, py, pz

    real(dp) :: dz2, dyz2, dist2, kern, ierr
    integer :: s, g_x, g_y, g_z, x, y, z, i, pencil

    s = blockIdx%x
    g_z = box(3, s) + blockIdx%y - 1
    if (g_z > box(6, s)) return
    z = local_index(g_z, gz, oz, nz, pz)
    if (z < 1) return

    dz2 = (src(3, s) - g_z*dz)**2
    if (dz2 > r_cut2) return
    do g_y = box(2, s) + threadIdx%x - 1, box(5, s), blockDim%x
      y = local_index(g_y, gy, oy, ny, py)
      if (y < 1) cycle
      dyz2 = dz2 + (src(2, s) - g_y*dy)**2
      if (dyz2 > r_cut2) cycle
      i = mod(y - 1, SZ) + 1
      pencil = z + ((y - 1)/SZ)*n_i_pad
      do g_x = box(1, s), box(4, s)
        x = local_index(g_x, gx, ox, nx, px)
        if (x < 1) cycle
        dist2 = dyz2 + (src(1, s) - g_x*dx)**2
        if (dist2 > r_cut2) cycle
        kern = exp(-dist2*inv_eps2)
        ierr = atomicadd(a(i, x, pencil), src(4, s)*kern)
        ierr = atomicadd(b(i, x, pencil), src(5, s)*kern)
        ierr = atomicadd(c(i, x, pencil), src(6, s)*kern)
      end do
    end do
  end subroutine add_gaussian_sources

end module m_cuda_kernels_fieldops