Unverified Commit 63861e82 authored by Dion Häfner's avatar Dion Häfner Committed by GitHub

Merge pull request #76 from team-ocean/mpi-free

Free MPI comm after use
parents 679f9c2b fbe314f4
...@@ -24,13 +24,13 @@ COMPONENTS = ['numpy', 'numpy-mpi', 'bohrium', 'bohrium-opencl', 'bohrium-cuda', ...@@ -24,13 +24,13 @@ COMPONENTS = ['numpy', 'numpy-mpi', 'bohrium', 'bohrium-opencl', 'bohrium-cuda',
STATIC_SETTINGS = '-v debug -s nx {nx} -s ny {ny} -s nz {nz} -s default_float_type {float_type} --timesteps {timesteps}' STATIC_SETTINGS = '-v debug -s nx {nx} -s ny {ny} -s nz {nz} -s default_float_type {float_type} --timesteps {timesteps}'
BENCHMARK_COMMANDS = { BENCHMARK_COMMANDS = {
'numpy': '{python} {filename} -b numpy ' + STATIC_SETTINGS, 'numpy': '{python} {filename} -b numpy ' + STATIC_SETTINGS,
'numpy-mpi': '{mpiexec} -n {nproc} -- {python} {filename} -b numpy -n {decomp} ' + STATIC_SETTINGS, 'numpy-mpi': '{mpiexec} -n {nproc} {python} {filename} -b numpy -n {decomp} ' + STATIC_SETTINGS,
'bohrium': 'OMP_NUM_THREADS={nproc} BH_STACK=openmp BH_OPENMP_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS, 'bohrium': 'OMP_NUM_THREADS={nproc} BH_STACK=openmp BH_OPENMP_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS,
'bohrium-opencl': 'BH_STACK=opencl BH_OPENCL_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS, 'bohrium-opencl': 'BH_STACK=opencl BH_OPENCL_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS,
'bohrium-cuda': 'BH_STACK=cuda BH_CUDA_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS, 'bohrium-cuda': 'BH_STACK=cuda BH_CUDA_PROF=1 {python} {filename} -b bohrium ' + STATIC_SETTINGS,
'bohrium-mpi': 'OMP_NUM_THREADS=1 {mpiexec} -n {nproc} -- {python} {filename} -b bohrium -n {decomp} ' + STATIC_SETTINGS, 'bohrium-mpi': 'OMP_NUM_THREADS=1 {mpiexec} -n {nproc} {python} {filename} -b bohrium -n {decomp} ' + STATIC_SETTINGS,
'fortran': '{python} {filename} --fortran {fortran_library} ' + STATIC_SETTINGS, 'fortran': '{python} {filename} --fortran {fortran_library} ' + STATIC_SETTINGS,
'fortran-mpi': '{mpiexec} -n {nproc} -- {python} {filename} --fortran {fortran_library} -n {decomp} ' + STATIC_SETTINGS 'fortran-mpi': '{mpiexec} -n {nproc} {python} {filename} --fortran {fortran_library} -n {decomp} ' + STATIC_SETTINGS
} }
SLURM_COMMANDS = { SLURM_COMMANDS = {
'numpy': 'srun --ntasks 1 --cpus-per-task {nproc} -- {python} {filename} -b numpy ' + STATIC_SETTINGS, 'numpy': 'srun --ntasks 1 --cpus-per-task {nproc} -- {python} {filename} -b numpy ' + STATIC_SETTINGS,
......
...@@ -56,6 +56,7 @@ def test_gather(backend): ...@@ -56,6 +56,7 @@ def test_gather(backend):
[1., 1., 1., 1., 3., 3., 3., 3.], [1., 1., 1., 1., 3., 3., 3., 3.],
[1., 1., 1., 1., 3., 3., 3., 3.]] [1., 1., 1., 1., 3., 3., 3., 3.]]
)) ))
else: else:
rs.num_proc = (2, 2) rs.num_proc = (2, 2)
...@@ -128,6 +129,7 @@ def test_scatter(backend): ...@@ -128,6 +129,7 @@ def test_scatter(backend):
for proc, idx in enumerate(proc_slices): for proc, idx in enumerate(proc_slices):
comm.Recv(res, proc) comm.Recv(res, proc)
assert np.array_equal(res, global_arr[idx]) assert np.array_equal(res, global_arr[idx])
else: else:
rs.num_proc = (2, 2) rs.num_proc = (2, 2)
...@@ -203,6 +205,7 @@ def test_acc(backend): ...@@ -203,6 +205,7 @@ def test_acc(backend):
) )
np.testing.assert_allclose(sim.state.psi / scale, other_psi / scale, rtol=0, atol=1e-5) np.testing.assert_allclose(sim.state.psi / scale, other_psi / scale, rtol=0, atol=1e-5)
else: else:
rs.num_proc = (2, 2) rs.num_proc = (2, 2)
......
...@@ -134,6 +134,13 @@ def streamfunction_init(vs): ...@@ -134,6 +134,13 @@ def streamfunction_init(vs):
@veros_method @veros_method
def _ascii_map(vs, boundary_map): def _ascii_map(vs, boundary_map):
def _get_char(c):
if c == 0:
return '.'
if c < 0:
return '#'
return str(c % 10)
map_string = '' map_string = ''
linewidth = 100 linewidth = 100
imt = vs.nx + 4 imt = vs.nx + 4
...@@ -150,8 +157,8 @@ def _ascii_map(vs, boundary_map): ...@@ -150,8 +157,8 @@ def _ascii_map(vs, boundary_map):
map_string += ''.join(['{:5d}'.format(istart + i + 1 - 2) for i in range(1, iline + 1, 5)]) map_string += ''.join(['{:5d}'.format(istart + i + 1 - 2) for i in range(1, iline + 1, 5)])
map_string += '\n' map_string += '\n'
for j in range(vs.ny + 3, -1, -1): for j in range(vs.ny + 3, -1, -1):
map_string += '{:3d} '.format(j) + ''.join([str(boundary_map[istart + i - 2, j] % 10) map_string += '{:3d} '.format(j)
if boundary_map[istart + i - 2, j] >= 0 else '*' for i in range(2, iline + 2)]) map_string += ''.join([_get_char(boundary_map[istart + i - 2, j]) for i in range(2, iline + 2)])
map_string += '\n' map_string += '\n'
map_string += ''.join(['{:5d}'.format(istart + i + 1 - 2) for i in range(1, iline + 1, 5)]) map_string += ''.join(['{:5d}'.format(istart + i + 1 - 2) for i in range(1, iline + 1, 5)])
map_string += '\n' map_string += '\n'
......
...@@ -284,31 +284,37 @@ def exchange_cyclic_boundaries(vs, arr): ...@@ -284,31 +284,37 @@ def exchange_cyclic_boundaries(vs, arr):
def _reduce(vs, arr, op, axis=None): def _reduce(vs, arr, op, axis=None):
if axis is None: if axis is None:
comm = rs.mpi_comm comm = rs.mpi_comm
disconnect_comm = False
else: else:
assert axis in (0, 1) assert axis in (0, 1)
pi = proc_rank_to_index(rst.proc_rank) pi = proc_rank_to_index(rst.proc_rank)
other_axis = 1 - axis other_axis = 1 - axis
comm = rs.mpi_comm.Split(pi[other_axis], rst.proc_rank) comm = rs.mpi_comm.Split(pi[other_axis], rst.proc_rank)
disconnect_comm = True
if np.isscalar(arr): try:
squeeze = True if np.isscalar(arr):
arr = np.array([arr]) squeeze = True
else: arr = np.array([arr])
squeeze = False else:
squeeze = False
arr = ascontiguousarray(arr) arr = ascontiguousarray(arr)
res = np.empty_like(arr) res = np.empty_like(arr)
comm.Allreduce( comm.Allreduce(
get_array_buffer(vs, arr), get_array_buffer(vs, arr),
get_array_buffer(vs, res), get_array_buffer(vs, res),
op=op op=op
) )
if squeeze: if squeeze:
res = res[0] res = res[0]
return res return res
finally:
if disconnect_comm:
comm.Disconnect()
@dist_context_only @dist_context_only
......
...@@ -248,12 +248,12 @@ class GlobalFlexibleResolutionSetup(VerosSetup): ...@@ -248,12 +248,12 @@ class GlobalFlexibleResolutionSetup(VerosSetup):
taux_raw = self._get_data(vs, 'tau_x', idx=data_subset) taux_raw = self._get_data(vs, 'tau_x', idx=data_subset)
taux_data = veros.tools.interpolate((xt_forc, yt_forc, np.arange(12)), taux_data = veros.tools.interpolate((xt_forc, yt_forc, np.arange(12)),
taux_raw, time_grid) taux_raw, time_grid)
vs.taux[2:-2, 2:-2, :] = taux_data / vs.rho_0 vs.taux[2:-2, 2:-2, :] = taux_data
tauy_raw = self._get_data(vs, 'tau_y', idx=data_subset) tauy_raw = self._get_data(vs, 'tau_y', idx=data_subset)
tauy_data = veros.tools.interpolate((xt_forc, yt_forc, np.arange(12)), tauy_data = veros.tools.interpolate((xt_forc, yt_forc, np.arange(12)),
tauy_raw, time_grid) tauy_raw, time_grid)
vs.tauy[2:-2, 2:-2, :] = tauy_data / vs.rho_0 vs.tauy[2:-2, 2:-2, :] = tauy_data
enforce_boundaries(vs, vs.taux) enforce_boundaries(vs, vs.taux)
enforce_boundaries(vs, vs.tauy) enforce_boundaries(vs, vs.tauy)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment