Unverified Commit 63861e82 authored by Dion Häfner's avatar Dion Häfner Committed by GitHub

Merge pull request #76 from team-ocean/mpi-free

Free MPI comm after use
parents 679f9c2b fbe314f4
......@@ -24,13 +24,13 @@ COMPONENTS = ['numpy', 'numpy-mpi', 'bohrium', 'bohrium-opencl', 'bohrium-cuda',
STATIC_SETTINGS = '-v debug -s nx {nx} -s ny {ny} -s nz {nz} -s default_float_type {float_type} --timesteps {timesteps}'
BENCHMARK_COMMANDS = {
'numpy': '{python} {filename} -b numpy ' + STATIC_SETTINGS,
'numpy-mpi': '{mpiexec} -n {nproc} -- {python} {filename} -b numpy -n {decomp} ' + STATIC_SETTINGS,
'numpy-mpi': '{mpiexec} -n {nproc} {python} {filename} -b numpy -n {decomp} ' + STATIC_SETTINGS,
'bohrium': 'OMP_NUM_THREADS={nproc} BH_STACK=openmp BH_OPENMP_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS,
'bohrium-opencl': 'BH_STACK=opencl BH_OPENCL_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS,
'bohrium-cuda': 'BH_STACK=cuda BH_CUDA_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS,
'bohrium-mpi': 'OMP_NUM_THREADS=1 {mpiexec} -n {nproc} -- {python} {filename} -b bohrium -n {decomp} ' + STATIC_SETTINGS,
'bohrium-mpi': 'OMP_NUM_THREADS=1 {mpiexec} -n {nproc} {python} {filename} -b bohrium -n {decomp} ' + STATIC_SETTINGS,
'fortran': '{python} {filename} --fortran {fortran_library} ' + STATIC_SETTINGS,
'fortran-mpi': '{mpiexec} -n {nproc} -- {python} {filename} --fortran {fortran_library} -n {decomp} ' + STATIC_SETTINGS
'fortran-mpi': '{mpiexec} -n {nproc} {python} {filename} --fortran {fortran_library} -n {decomp} ' + STATIC_SETTINGS
}
SLURM_COMMANDS = {
'numpy': 'srun --ntasks 1 --cpus-per-task {nproc} -- {python} {filename} -b numpy ' + STATIC_SETTINGS,
......
......@@ -56,6 +56,7 @@ def test_gather(backend):
[1., 1., 1., 1., 3., 3., 3., 3.],
[1., 1., 1., 1., 3., 3., 3., 3.]]
))
else:
rs.num_proc = (2, 2)
......@@ -128,6 +129,7 @@ def test_scatter(backend):
for proc, idx in enumerate(proc_slices):
comm.Recv(res, proc)
assert np.array_equal(res, global_arr[idx])
else:
rs.num_proc = (2, 2)
......@@ -203,6 +205,7 @@ def test_acc(backend):
)
np.testing.assert_allclose(sim.state.psi / scale, other_psi / scale, rtol=0, atol=1e-5)
else:
rs.num_proc = (2, 2)
......
......@@ -134,6 +134,13 @@ def streamfunction_init(vs):
@veros_method
def _ascii_map(vs, boundary_map):
def _get_char(c):
if c == 0:
return '.'
if c < 0:
return '#'
return str(c % 10)
map_string = ''
linewidth = 100
imt = vs.nx + 4
......@@ -150,8 +157,8 @@ def _ascii_map(vs, boundary_map):
map_string += ''.join(['{:5d}'.format(istart + i + 1 - 2) for i in range(1, iline + 1, 5)])
map_string += '\n'
for j in range(vs.ny + 3, -1, -1):
map_string += '{:3d} '.format(j) + ''.join([str(boundary_map[istart + i - 2, j] % 10)
if boundary_map[istart + i - 2, j] >= 0 else '*' for i in range(2, iline + 2)])
map_string += '{:3d} '.format(j)
map_string += ''.join([_get_char(boundary_map[istart + i - 2, j]) for i in range(2, iline + 2)])
map_string += '\n'
map_string += ''.join(['{:5d}'.format(istart + i + 1 - 2) for i in range(1, iline + 1, 5)])
map_string += '\n'
......
......@@ -284,12 +284,15 @@ def exchange_cyclic_boundaries(vs, arr):
def _reduce(vs, arr, op, axis=None):
if axis is None:
comm = rs.mpi_comm
disconnect_comm = False
else:
assert axis in (0, 1)
pi = proc_rank_to_index(rst.proc_rank)
other_axis = 1 - axis
comm = rs.mpi_comm.Split(pi[other_axis], rst.proc_rank)
disconnect_comm = True
try:
if np.isscalar(arr):
squeeze = True
arr = np.array([arr])
......@@ -309,6 +312,9 @@ def _reduce(vs, arr, op, axis=None):
res = res[0]
return res
finally:
if disconnect_comm:
comm.Disconnect()
@dist_context_only
......
......@@ -248,12 +248,12 @@ class GlobalFlexibleResolutionSetup(VerosSetup):
taux_raw = self._get_data(vs, 'tau_x', idx=data_subset)
taux_data = veros.tools.interpolate((xt_forc, yt_forc, np.arange(12)),
taux_raw, time_grid)
vs.taux[2:-2, 2:-2, :] = taux_data / vs.rho_0
vs.taux[2:-2, 2:-2, :] = taux_data
tauy_raw = self._get_data(vs, 'tau_y', idx=data_subset)
tauy_data = veros.tools.interpolate((xt_forc, yt_forc, np.arange(12)),
tauy_raw, time_grid)
vs.tauy[2:-2, 2:-2, :] = tauy_data / vs.rho_0
vs.tauy[2:-2, 2:-2, :] = tauy_data
enforce_boundaries(vs, vs.taux)
enforce_boundaries(vs, vs.tauy)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment