Skip to content

Optimize ACC port of atm_advance_scalars_work:4857 #1521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
abishekg7 wants to merge 6 commits into
base: develop
Choose a base branch
from
+49 −46
Draft

Optimize ACC port of atm_advance_scalars_work:4857 #1521

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a1409f6
Adding timer atm_advance_scalars_4857
abishekg7 Jun 18, 2026
7686fa2
Fix timer
abishekg7 Jun 18, 2026
74c2754
Opt 1 + macros
abishekg7 Sep 22, 2026
a99abdd
Opt 2
abishekg7 Sep 24, 2026
b818d31
Opt 3 + macro, scalar_tend_column can be replaced by a scalar local t…
abishekg7 Sep 24, 2026
991126c
Opt 4 + macro, remove loops that zero scalar_tend_column
abishekg7 Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 49 additions & 46 deletions src/core_atmosphere/dynamics/mpas_atm_time_integration.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,21 @@
#ifdef MPAS_OPENACC
#define MPAS_ACC_TIMER_START(X) call mpas_timer_start(X)
#define MPAS_ACC_TIMER_STOP(X) call mpas_timer_stop(X)
#define NEDGESONCELL_INNER_LOOP() do i=1,nEdgesOnCell(iCell)
#define NEDGESONCELL_OUTER_LOOP()
#define ACC_LOOP_SEQ() !$acc loop seq
#define SPLIT_K_ISCALAR_LOOP()
#define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column_tmp
#define ZERO_SCALAR_TEND_COLUMN() scalar_tend_column_tmp = 0.0_RKIND
#else
#define MPAS_ACC_TIMER_START(X)
#define MPAS_ACC_TIMER_STOP(X)
#define NEDGESONCELL_OUTER_LOOP() do i=1,nEdgesOnCell(iCell)
#define NEDGESONCELL_INNER_LOOP()
#define ACC_LOOP_SEQ()
#define SPLIT_K_ISCALAR_LOOP() end do; end do; do k=1,nVertLevels; do iScalar=1,num_scalars
#define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column(iScalar,k)
#define ZERO_SCALAR_TEND_COLUMN()
#endif

module atm_time_integration
Expand Down Expand Up @@ -4726,6 +4738,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
real (kind=RKIND) :: weight_time_old, weight_time_new
real (kind=RKIND), dimension(num_scalars,nVertLevels) :: scalar_tend_column ! local storage to accumulate tendency
real (kind=RKIND) :: u_direction, u_positive, u_negative
real (kind=RKIND) :: scalar_tend_column_tmp

flux4(q_im2, q_im1, q_i, q_ip1, ua) = &
ua*( 7.*(q_i + q_im1) - (q_ip1 + q_im2) )/12.0
Expand Down Expand Up @@ -4850,51 +4863,12 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
! and add vertical flux divergence in update.
!

MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]')
!$acc enter data create(scalar_tend_column)
MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]')

call mpas_timer_start('atm_advance_scalars_4857')
!$acc parallel wait
!$acc loop gang worker private(scalar_tend_column, wdtn)
!$acc loop gang worker private(wdtn)
do iCell=cellSolveStart,cellSolveEnd

if(bdyMaskCell(iCell) <= nRelaxZone) then ! specified zone for regional_MPAS is not updated in this routine

!$acc loop vector collapse(2)
do k=1,nVertLevels
do iScalar=1,num_scalars
scalar_tend_column(iScalar,k) = 0.0_RKIND
#ifndef DO_PHYSICS
scalar_tend_save(iScalar,k,iCell) = 0.0_RKIND ! testing purposes - we have no sources or sinks
#endif
end do
end do

!$acc loop seq
do i=1,nEdgesOnCell(iCell)
iEdge = edgesOnCell(i,iCell)

! here we add the horizontal flux divergence into the scalar tendency.
! note that the scalar tendency is modified.
!$acc loop vector collapse(2)
do k=1,nVertLevels
do iScalar=1,num_scalars
scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) &
- edgesOnCell_sign(i,iCell) * uhAvg(k,iEdge)*horiz_flux_arr(iScalar,k,iEdge)
end do
end do

end do

!$acc loop vector collapse(2)
do k=1,nVertLevels
do iScalar=1,num_scalars
scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) * invAreaCell(iCell) &
+ scalar_tend_save(iScalar,k,iCell)
end do
end do


!
! vertical flux divergence and update of the scalars
!
Expand All @@ -4918,12 +4892,43 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
end do
end do

#ifndef MPAS_OPENACC
do k=1,nVertLevels
do iScalar=1,num_scalars
SCALAR_TEND_COLUMN(iScalar,k) = 0.0_RKIND
end do
end do
#endif


NEDGESONCELL_OUTER_LOOP()

!$acc loop vector collapse(2)
do k=1,nVertLevels
do iScalar=1,num_scalars
do iScalar=1,num_scalars

ZERO_SCALAR_TEND_COLUMN()
! here we add the horizontal flux divergence into the scalar tendency.
! note that the scalar tendency is modified.
ACC_LOOP_SEQ()
NEDGESONCELL_INNER_LOOP()
iEdge = edgesOnCell(i,iCell)
SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) &
- edgesOnCell_sign(i,iCell) * uhAvg(k,iEdge)*horiz_flux_arr(iScalar,k,iEdge)
end do

SPLIT_K_ISCALAR_LOOP()
#ifndef DO_PHYSICS
scalar_tend_save(iScalar,k,iCell) = 0.0_RKIND ! testing purposes - we have no sources or sinks
#endif
SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) * invAreaCell(iCell) &
+ scalar_tend_save(iScalar,k,iCell)

SPLIT_K_ISCALAR_LOOP()

rho_zz_new_inv = 1.0_RKIND / (weight_time_old*rho_zz_old(k,iCell) + weight_time_new*rho_zz_new(k,iCell))
scalar_new(iScalar,k,iCell) = ( scalar_old(iScalar,k,iCell)*rho_zz_old(k,iCell) &
+ dt*( scalar_tend_column(iScalar,k) -rdnw(k)*(wdtn(iScalar,k+1)-wdtn(iScalar,k)) ) ) * rho_zz_new_inv
+ dt*( SCALAR_TEND_COLUMN(iScalar,k) -rdnw(k)*(wdtn(iScalar,k+1)-wdtn(iScalar,k)) ) ) * rho_zz_new_inv
end do
end do

Expand All @@ -4932,9 +4937,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
end do
!$acc end parallel

MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]')
!$acc exit data delete(scalar_tend_column)
MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]')
call mpas_timer_stop('atm_advance_scalars_4857')

end subroutine atm_advance_scalars_work

Expand Down