From a1409f6d292f216fb90ac47631bef6bde8987595 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 18 Jun 2026 15:49:51 -0600 Subject: [PATCH 1/6] Adding timer atm_advance_scalars_4857 --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 227fbde862..bad87b79d5 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4854,6 +4854,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & !$acc enter data create(scalar_tend_column) MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]') + MPAS_ACC_TIMER_START('atm_advance_scalars_4857') !$acc parallel wait !$acc loop gang worker private(scalar_tend_column, wdtn) do iCell=cellSolveStart,cellSolveEnd @@ -4932,6 +4933,8 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end do !$acc end parallel + MPAS_ACC_TIMER_STOP('atm_advance_scalars_4857') + MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]') !$acc exit data delete(scalar_tend_column) MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]') From 7686fa29384e994199760756232a3664787e057d Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 18 Jun 2026 16:04:20 -0600 Subject: [PATCH 2/6] Fix timer --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index bad87b79d5..ef347f55e9 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -4854,7 +4854,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & !$acc enter data create(scalar_tend_column) MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]') - MPAS_ACC_TIMER_START('atm_advance_scalars_4857') + call mpas_timer_start('atm_advance_scalars_4857') !$acc parallel wait !$acc loop gang worker private(scalar_tend_column, wdtn) do iCell=cellSolveStart,cellSolveEnd @@ -4933,7 +4933,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end do !$acc end parallel - MPAS_ACC_TIMER_STOP('atm_advance_scalars_4857') + call mpas_timer_stop('atm_advance_scalars_4857') MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]') !$acc exit data delete(scalar_tend_column) From 74c27543b5e7ddcd2df230df70bb3d28c9f5fc04 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Tue, 22 Sep 2026 16:59:20 -0600 Subject: [PATCH 3/6] Opt 1 + macros --- .../dynamics/mpas_atm_time_integration.F | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index ef347f55e9..8d76ac1a33 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -9,9 +9,15 @@ #ifdef MPAS_OPENACC #define MPAS_ACC_TIMER_START(X) call mpas_timer_start(X) #define MPAS_ACC_TIMER_STOP(X) call mpas_timer_stop(X) +#define NEDGESONCELL_INNER_LOOP() do i=1,nEdgesOnCell(iCell) +#define NEDGESONCELL_OUTER_LOOP() +#define ACC_LOOP_SEQ() !$acc loop seq #else #define MPAS_ACC_TIMER_START(X) #define MPAS_ACC_TIMER_STOP(X) +#define NEDGESONCELL_OUTER_LOOP() do i=1,nEdgesOnCell(iCell) +#define NEDGESONCELL_INNER_LOOP() +#define ACC_LOOP_SEQ() #endif module atm_time_integration @@ -4871,20 +4877,23 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end do end do - !$acc loop seq - do i=1,nEdgesOnCell(iCell) - iEdge = edgesOnCell(i,iCell) + + NEDGESONCELL_OUTER_LOOP() + + !$acc loop vector collapse(2) + do k=1,nVertLevels + do iScalar=1,num_scalars ! here we add the horizontal flux divergence into the scalar tendency. ! note that the scalar tendency is modified. - !$acc loop vector collapse(2) - do k=1,nVertLevels - do iScalar=1,num_scalars + ACC_LOOP_SEQ() + NEDGESONCELL_INNER_LOOP() + iEdge = edgesOnCell(i,iCell) scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) & - edgesOnCell_sign(i,iCell) * uhAvg(k,iEdge)*horiz_flux_arr(iScalar,k,iEdge) - end do end do - + + end do end do !$acc loop vector collapse(2) From a99abdd4ad641835fcfcc235de46dd9a5a92a254 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 24 Sep 2026 12:05:45 -0600 Subject: [PATCH 4/6] Opt 2 + macros - experimental --- .../dynamics/mpas_atm_time_integration.F | 64 +++++++++---------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 8d76ac1a33..14289dbe09 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -12,12 +12,14 @@ #define NEDGESONCELL_INNER_LOOP() do i=1,nEdgesOnCell(iCell) #define NEDGESONCELL_OUTER_LOOP() #define ACC_LOOP_SEQ() !$acc loop seq +#define SPLIT_K_ISCALAR_LOOP() #else #define MPAS_ACC_TIMER_START(X) #define MPAS_ACC_TIMER_STOP(X) #define NEDGESONCELL_OUTER_LOOP() do i=1,nEdgesOnCell(iCell) #define NEDGESONCELL_INNER_LOOP() #define ACC_LOOP_SEQ() +#define SPLIT_K_ISCALAR_LOOP() end do; end do; do k=1,nVertLevels; do iScalar=1,num_scalars #endif module atm_time_integration @@ -4732,6 +4734,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & real (kind=RKIND) :: weight_time_old, weight_time_new real (kind=RKIND), dimension(num_scalars,nVertLevels) :: scalar_tend_column ! local storage to accumulate tendency real (kind=RKIND) :: u_direction, u_positive, u_negative + real (kind=RKIND) :: scalar_tend_column_tmp flux4(q_im2, q_im1, q_i, q_ip1, ua) = & ua*( 7.*(q_i + q_im1) - (q_ip1 + q_im2) )/12.0 @@ -4866,6 +4869,28 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & do iCell=cellSolveStart,cellSolveEnd if(bdyMaskCell(iCell) <= nRelaxZone) then ! specified zone for regional_MPAS is not updated in this routine + ! + ! vertical flux divergence and update of the scalars + ! + + !$acc loop vector + do iScalar=1,num_scalars + wdtn(iScalar,1) = 0.0 + wdtn(iScalar,2) = wwAvg(2,iCell)*(fnm(2)*scalar_new(iScalar,2,iCell)+fnp(2)*scalar_new(iScalar,2-1,iCell)) + wdtn(iScalar,nVertLevels) = wwAvg(nVertLevels,iCell) * & + ( fnm(nVertLevels)*scalar_new(iScalar,nVertLevels,iCell) & + +fnp(nVertLevels)*scalar_new(iScalar,nVertLevels-1,iCell) ) + wdtn(iScalar,nVertLevels+1) = 0.0 + end do + + !$acc loop vector collapse(2) + do k=3,nVertLevels-1 + do iScalar=1,num_scalars + wdtn(iScalar,k) = flux3( scalar_new(iScalar,k-2,iCell),scalar_new(iScalar,k-1,iCell), & + scalar_new(iScalar,k ,iCell),scalar_new(iScalar,k+1,iCell), & + wwAvg(k,iCell), coef_3rd_order ) + end do + end do !$acc loop vector collapse(2) do k=1,nVertLevels @@ -4893,44 +4918,13 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & - edgesOnCell_sign(i,iCell) * uhAvg(k,iEdge)*horiz_flux_arr(iScalar,k,iEdge) end do - end do - end do - - !$acc loop vector collapse(2) - do k=1,nVertLevels - do iScalar=1,num_scalars + SPLIT_K_ISCALAR_LOOP() + scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) * invAreaCell(iCell) & + scalar_tend_save(iScalar,k,iCell) - end do - end do - - - ! - ! vertical flux divergence and update of the scalars - ! - - !$acc loop vector - do iScalar=1,num_scalars - wdtn(iScalar,1) = 0.0 - wdtn(iScalar,2) = wwAvg(2,iCell)*(fnm(2)*scalar_new(iScalar,2,iCell)+fnp(2)*scalar_new(iScalar,2-1,iCell)) - wdtn(iScalar,nVertLevels) = wwAvg(nVertLevels,iCell) * & - ( fnm(nVertLevels)*scalar_new(iScalar,nVertLevels,iCell) & - +fnp(nVertLevels)*scalar_new(iScalar,nVertLevels-1,iCell) ) - wdtn(iScalar,nVertLevels+1) = 0.0 - end do - - !$acc loop vector collapse(2) - do k=3,nVertLevels-1 - do iScalar=1,num_scalars - wdtn(iScalar,k) = flux3( scalar_new(iScalar,k-2,iCell),scalar_new(iScalar,k-1,iCell), & - scalar_new(iScalar,k ,iCell),scalar_new(iScalar,k+1,iCell), & - wwAvg(k,iCell), coef_3rd_order ) - end do - end do + + SPLIT_K_ISCALAR_LOOP() - !$acc loop vector collapse(2) - do k=1,nVertLevels - do iScalar=1,num_scalars rho_zz_new_inv = 1.0_RKIND / (weight_time_old*rho_zz_old(k,iCell) + weight_time_new*rho_zz_new(k,iCell)) scalar_new(iScalar,k,iCell) = ( scalar_old(iScalar,k,iCell)*rho_zz_old(k,iCell) & + dt*( scalar_tend_column(iScalar,k) -rdnw(k)*(wdtn(iScalar,k+1)-wdtn(iScalar,k)) ) ) * rho_zz_new_inv From b818d316bcbf93b440064e0745baf95335321716 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 24 Sep 2026 13:16:34 -0600 Subject: [PATCH 5/6] Opt 3 + macro, scalar_tend_column can be replaced by a scalar local to each gang --- .../dynamics/mpas_atm_time_integration.F | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 14289dbe09..a8fb90a54e 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -13,6 +13,7 @@ #define NEDGESONCELL_OUTER_LOOP() #define ACC_LOOP_SEQ() !$acc loop seq #define SPLIT_K_ISCALAR_LOOP() +#define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column_tmp #else #define MPAS_ACC_TIMER_START(X) #define MPAS_ACC_TIMER_STOP(X) @@ -20,6 +21,7 @@ #define NEDGESONCELL_INNER_LOOP() #define ACC_LOOP_SEQ() #define SPLIT_K_ISCALAR_LOOP() end do; end do; do k=1,nVertLevels; do iScalar=1,num_scalars +#define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column(iScalar,k) #endif module atm_time_integration @@ -4859,13 +4861,9 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & ! and add vertical flux divergence in update. ! - MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]') - !$acc enter data create(scalar_tend_column) - MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]') - call mpas_timer_start('atm_advance_scalars_4857') !$acc parallel wait - !$acc loop gang worker private(scalar_tend_column, wdtn) + !$acc loop gang worker private(wdtn) do iCell=cellSolveStart,cellSolveEnd if(bdyMaskCell(iCell) <= nRelaxZone) then ! specified zone for regional_MPAS is not updated in this routine @@ -4895,7 +4893,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & !$acc loop vector collapse(2) do k=1,nVertLevels do iScalar=1,num_scalars - scalar_tend_column(iScalar,k) = 0.0_RKIND + SCALAR_TEND_COLUMN(iScalar,k) = 0.0_RKIND #ifndef DO_PHYSICS scalar_tend_save(iScalar,k,iCell) = 0.0_RKIND ! testing purposes - we have no sources or sinks #endif @@ -4914,20 +4912,20 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & ACC_LOOP_SEQ() NEDGESONCELL_INNER_LOOP() iEdge = edgesOnCell(i,iCell) - scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) & + SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) & - edgesOnCell_sign(i,iCell) * uhAvg(k,iEdge)*horiz_flux_arr(iScalar,k,iEdge) end do SPLIT_K_ISCALAR_LOOP() - scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) * invAreaCell(iCell) & + SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) * invAreaCell(iCell) & + scalar_tend_save(iScalar,k,iCell) SPLIT_K_ISCALAR_LOOP() rho_zz_new_inv = 1.0_RKIND / (weight_time_old*rho_zz_old(k,iCell) + weight_time_new*rho_zz_new(k,iCell)) scalar_new(iScalar,k,iCell) = ( scalar_old(iScalar,k,iCell)*rho_zz_old(k,iCell) & - + dt*( scalar_tend_column(iScalar,k) -rdnw(k)*(wdtn(iScalar,k+1)-wdtn(iScalar,k)) ) ) * rho_zz_new_inv + + dt*( SCALAR_TEND_COLUMN(iScalar,k) -rdnw(k)*(wdtn(iScalar,k+1)-wdtn(iScalar,k)) ) ) * rho_zz_new_inv end do end do @@ -4938,10 +4936,6 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & call mpas_timer_stop('atm_advance_scalars_4857') - MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]') - !$acc exit data delete(scalar_tend_column) - MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]') - end subroutine atm_advance_scalars_work From 991126c00acdae9664bc9b065864aba35649c3d5 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 24 Sep 2026 16:11:18 -0600 Subject: [PATCH 6/6] Opt 4 + macro, remove loops that zero scalar_tend_column --- .../dynamics/mpas_atm_time_integration.F | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index a8fb90a54e..6584d72a9d 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -14,6 +14,7 @@ #define ACC_LOOP_SEQ() !$acc loop seq #define SPLIT_K_ISCALAR_LOOP() #define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column_tmp +#define ZERO_SCALAR_TEND_COLUMN() scalar_tend_column_tmp = 0.0_RKIND #else #define MPAS_ACC_TIMER_START(X) #define MPAS_ACC_TIMER_STOP(X) @@ -22,6 +23,7 @@ #define ACC_LOOP_SEQ() #define SPLIT_K_ISCALAR_LOOP() end do; end do; do k=1,nVertLevels; do iScalar=1,num_scalars #define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column(iScalar,k) +#define ZERO_SCALAR_TEND_COLUMN() #endif module atm_time_integration @@ -4890,15 +4892,13 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end do end do - !$acc loop vector collapse(2) +#ifndef MPAS_OPENACC do k=1,nVertLevels do iScalar=1,num_scalars SCALAR_TEND_COLUMN(iScalar,k) = 0.0_RKIND -#ifndef DO_PHYSICS - scalar_tend_save(iScalar,k,iCell) = 0.0_RKIND ! testing purposes - we have no sources or sinks -#endif end do end do +#endif NEDGESONCELL_OUTER_LOOP() @@ -4907,6 +4907,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & do k=1,nVertLevels do iScalar=1,num_scalars + ZERO_SCALAR_TEND_COLUMN() ! here we add the horizontal flux divergence into the scalar tendency. ! note that the scalar tendency is modified. ACC_LOOP_SEQ() @@ -4917,8 +4918,10 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, & end do SPLIT_K_ISCALAR_LOOP() - - SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) * invAreaCell(iCell) & +#ifndef DO_PHYSICS + scalar_tend_save(iScalar,k,iCell) = 0.0_RKIND ! testing purposes - we have no sources or sinks +#endif + SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) * invAreaCell(iCell) & + scalar_tend_save(iScalar,k,iCell) SPLIT_K_ISCALAR_LOOP()