diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F
index 227fbde862..6584d72a9d 100644
--- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F
+++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F
@@ -9,9 +9,21 @@
 #ifdef MPAS_OPENACC
 #define MPAS_ACC_TIMER_START(X) call mpas_timer_start(X)
 #define MPAS_ACC_TIMER_STOP(X) call mpas_timer_stop(X)
+#define NEDGESONCELL_INNER_LOOP()  do i=1,nEdgesOnCell(iCell)
+#define NEDGESONCELL_OUTER_LOOP()
+#define ACC_LOOP_SEQ() !$acc loop seq
+#define SPLIT_K_ISCALAR_LOOP()
+#define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column_tmp
+#define ZERO_SCALAR_TEND_COLUMN() scalar_tend_column_tmp = 0.0_RKIND
 #else
 #define MPAS_ACC_TIMER_START(X)
 #define MPAS_ACC_TIMER_STOP(X)
+#define NEDGESONCELL_OUTER_LOOP()  do i=1,nEdgesOnCell(iCell)
+#define NEDGESONCELL_INNER_LOOP()
+#define ACC_LOOP_SEQ()
+#define SPLIT_K_ISCALAR_LOOP()  end do; end do; do k=1,nVertLevels; do iScalar=1,num_scalars
+#define SCALAR_TEND_COLUMN(iScalar,k) scalar_tend_column(iScalar,k)
+#define ZERO_SCALAR_TEND_COLUMN()
 #endif
 
 module atm_time_integration
@@ -4726,6 +4738,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
       real (kind=RKIND) :: weight_time_old, weight_time_new
       real (kind=RKIND), dimension(num_scalars,nVertLevels) :: scalar_tend_column  ! local storage to accumulate tendency
       real (kind=RKIND) :: u_direction, u_positive, u_negative
+      real (kind=RKIND) :: scalar_tend_column_tmp
 
       flux4(q_im2, q_im1, q_i, q_ip1, ua) =                     &
           ua*( 7.*(q_i + q_im1) - (q_ip1 + q_im2) )/12.0
@@ -4850,51 +4863,12 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
       ! and add vertical flux divergence in update.
       !
 
-      MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]')
-      !$acc enter data create(scalar_tend_column)
-      MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]')
-
+      call mpas_timer_start('atm_advance_scalars_4857')
       !$acc parallel wait
-      !$acc loop gang worker private(scalar_tend_column, wdtn)
+      !$acc loop gang worker private(wdtn)
       do iCell=cellSolveStart,cellSolveEnd
 
          if(bdyMaskCell(iCell) <= nRelaxZone) then  ! specified zone for regional_MPAS is not updated in this routine
-
-            !$acc loop vector collapse(2)
-            do k=1,nVertLevels
-            do iScalar=1,num_scalars
-               scalar_tend_column(iScalar,k) = 0.0_RKIND
-#ifndef DO_PHYSICS
-               scalar_tend_save(iScalar,k,iCell) = 0.0_RKIND  !  testing purposes - we have no sources or sinks
-#endif
-            end do
-            end do
-
-            !$acc loop seq
-            do i=1,nEdgesOnCell(iCell)
-               iEdge = edgesOnCell(i,iCell)
-
-               ! here we add the horizontal flux divergence into the scalar tendency.
-               ! note that the scalar tendency is modified.
-               !$acc loop vector collapse(2)
-               do k=1,nVertLevels
-                  do iScalar=1,num_scalars
-                        scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) &
-                               - edgesOnCell_sign(i,iCell) * uhAvg(k,iEdge)*horiz_flux_arr(iScalar,k,iEdge)
-                  end do
-               end do
-
-            end do
-
-            !$acc loop vector collapse(2)
-            do k=1,nVertLevels
-               do iScalar=1,num_scalars
-                     scalar_tend_column(iScalar,k) = scalar_tend_column(iScalar,k) * invAreaCell(iCell) &
-                                                   + scalar_tend_save(iScalar,k,iCell)
-               end do
-            end do
-
-
             !
             !  vertical flux divergence and update of the scalars
             !
@@ -4918,12 +4892,43 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
                end do
             end do
 
+#ifndef MPAS_OPENACC
+            do k=1,nVertLevels
+            do iScalar=1,num_scalars
+               SCALAR_TEND_COLUMN(iScalar,k) = 0.0_RKIND
+            end do
+            end do
+#endif
+
+
+            NEDGESONCELL_OUTER_LOOP()
+
             !$acc loop vector collapse(2)
             do k=1,nVertLevels
-               do iScalar=1,num_scalars
+            do iScalar=1,num_scalars
+
+               ZERO_SCALAR_TEND_COLUMN()
+               ! here we add the horizontal flux divergence into the scalar tendency.
+               ! note that the scalar tendency is modified.
+               ACC_LOOP_SEQ()
+               NEDGESONCELL_INNER_LOOP()
+                  iEdge = edgesOnCell(i,iCell)
+                        SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) &
+                               - edgesOnCell_sign(i,iCell) * uhAvg(k,iEdge)*horiz_flux_arr(iScalar,k,iEdge)
+               end do
+            
+               SPLIT_K_ISCALAR_LOOP()
+#ifndef DO_PHYSICS
+               scalar_tend_save(iScalar,k,iCell) = 0.0_RKIND  !  testing purposes - we have no sources or sinks
+#endif   
+               SCALAR_TEND_COLUMN(iScalar,k) = SCALAR_TEND_COLUMN(iScalar,k) * invAreaCell(iCell) &
+                                                   + scalar_tend_save(iScalar,k,iCell)
+   
+               SPLIT_K_ISCALAR_LOOP()
+
                   rho_zz_new_inv = 1.0_RKIND / (weight_time_old*rho_zz_old(k,iCell) + weight_time_new*rho_zz_new(k,iCell))
                   scalar_new(iScalar,k,iCell) = (   scalar_old(iScalar,k,iCell)*rho_zz_old(k,iCell) &
-                        + dt*( scalar_tend_column(iScalar,k) -rdnw(k)*(wdtn(iScalar,k+1)-wdtn(iScalar,k)) ) ) * rho_zz_new_inv
+                        + dt*( SCALAR_TEND_COLUMN(iScalar,k) -rdnw(k)*(wdtn(iScalar,k+1)-wdtn(iScalar,k)) ) ) * rho_zz_new_inv
                end do
             end do
 
@@ -4932,9 +4937,7 @@ subroutine atm_advance_scalars_work(nCells, num_scalars, dt, &
       end do
       !$acc end parallel
 
-      MPAS_ACC_TIMER_START('atm_advance_scalars [ACC_data_xfer]')
-      !$acc exit data delete(scalar_tend_column)
-      MPAS_ACC_TIMER_STOP('atm_advance_scalars [ACC_data_xfer]')
+      call mpas_timer_stop('atm_advance_scalars_4857')
 
    end subroutine atm_advance_scalars_work