fixed boundary comm w/ MPI

wilfonba · Feb 23, 2025 · 5992f4e · 5992f4e
1 parent 7c4d220
commit 5992f4e
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 9 deletions.
diff --git a/examples/2D_igr/case.py b/examples/2D_igr/case.py
@@ -41,8 +41,8 @@
     # "t_save": 2.5/100,
     "dt": dt,
     "t_step_start": 0,
-    "t_step_stop": 1,
-    "t_step_save": 1,
+    "t_step_stop": 500,
+    "t_step_save": 10,
     # Simulation Algorithm
     "model_eqns": 2,
     "alt_soundspeed": "F",

diff --git a/src/pre_process/m_mpi_proxy.fpp b/src/pre_process/m_mpi_proxy.fpp
@@ -482,7 +482,6 @@ contains
                                                           num_procs_y/), (/.true., &
                                                                            .true./), .false., MPI_COMM_CART, &
                                      ierr)
-
                 ! Finding corresponding Cartesian coordinates of the local
                 ! processor rank in newly declared cartesian communicator
                 call MPI_CART_COORDS(MPI_COMM_CART, proc_rank, 2, &
@@ -598,8 +597,6 @@ contains
         if (proc_coords(1) > 0 .or. (bc_x%beg == -1 .and. num_procs_x > 1)) then
             proc_coords(1) = proc_coords(1) - 1
             call MPI_CART_RANK(MPI_COMM_CART, proc_coords, bc_x%beg, ierr)
-            print *, proc_rank, bc_x 
-            print *
             proc_coords(1) = proc_coords(1) + 1
         end if
 
@@ -663,6 +660,8 @@ contains
 
         integer :: pack_offset, unpack_offset
 
+        real(wp), pointer :: p_send, p_recv
+
 #ifdef MFC_MPI
 
         buffer_counts = (/ &
@@ -723,8 +722,8 @@ contains
                             do k = 0, buff_size - 1
                                 do j = -buff_size, m + buff_size
                                     r = (i - 1) + v_size* &
-                                        ((j + 1) + (m + 2*1 + 1)* &
-                                         (k + 1*l))
+                                        ((j + buff_size) + (m + 2*buff_size + 1)* &
+                                         (k + buff_size*l))
                                     q_prims_buff_send(r) = q_prim_vf(i)%sf(j, k + pack_offset, l)
                                 end do
                             end do
@@ -734,7 +733,7 @@ contains
                     !$acc parallel loop collapse(4) gang vector default(present) private(r)
                     do i = 1, sys_size
                         do l = 0, buff_size - 1
-                            do k = -1, n + 1
+                            do k = -buff_size, n + buff_size
                                 do j = -buff_size, m + buff_size
                                     r = (i - 1) + v_size* &
                                         ((j + buff_size) + (m + 2*buff_size + 1)* &
@@ -748,7 +747,14 @@ contains
             end if
         #:endfor
 
+        p_send => q_prims_buff_send(0)
+        p_recv => q_prims_buff_recv(0)
 
+        call MPI_SENDRECV( &
+        p_send, buffer_count, mpi_p, dst_proc, send_tag, &
+        p_recv, buffer_count, mpi_p, src_proc, recv_tag, &
+        MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
+        
         ! Unpack Received Buffer
         #:for mpi_dir in [1, 2, 3]
             if (mpi_dir == ${mpi_dir}$) then

diff --git a/src/pre_process/m_perturbation.fpp b/src/pre_process/m_perturbation.fpp
@@ -616,7 +616,6 @@ contains
         type(scalar_field), dimension(sys_size) :: q_prim_vf
         real(wp), dimension(0:m, 0:n, 0:p, 1:sys_size) :: q_prim_temp
         integer :: i, j, k, l, q
-        print*, proc_rank, bcxb, bcxe
         do q = 1, elliptic_smoothing_iters
             if(bcxb >= -12) then
                 if(bcxb >= 0) then