Commit 2d8db34d authored by Thomas Jahns's avatar Thomas Jahns 🤸
Browse files

Make threshold for automatic vector to stripes conversion tunable.

parent 0d657ee9
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
if FC_MOD_UPPERCASE if FC_MOD_UPPERCASE
##UPPERCASE_MODULE_NAME_MARKER do not remove## ##UPPERCASE_MODULE_NAME_MARKER do not remove##
XT_XMAP_RENAME_MOD_NAME=XT_XMAP_RENAME
XT_REDIST_RENAME_MOD_NAME=XT_REDIST_RENAME XT_REDIST_RENAME_MOD_NAME=XT_REDIST_RENAME
XT_CONFIG_F_MOD_NAME=XT_CONFIG_F XT_CONFIG_F_MOD_NAME=XT_CONFIG_F
XT_SORT_MOD_NAME=XT_SORT XT_SORT_MOD_NAME=XT_SORT
...@@ -59,6 +60,7 @@ XT_UT_MOD_NAME=XT_UT ...@@ -59,6 +60,7 @@ XT_UT_MOD_NAME=XT_UT
YAXT_MOD_NAME=YAXT YAXT_MOD_NAME=YAXT
else else
##LOWERCASE_MODULE_NAME_MARKER do not remove## ##LOWERCASE_MODULE_NAME_MARKER do not remove##
XT_XMAP_RENAME_MOD_NAME=xt_xmap_rename
XT_REDIST_RENAME_MOD_NAME=xt_redist_rename XT_REDIST_RENAME_MOD_NAME=xt_redist_rename
XT_CONFIG_F_MOD_NAME=xt_config_f XT_CONFIG_F_MOD_NAME=xt_config_f
XT_SORT_MOD_NAME=xt_sort XT_SORT_MOD_NAME=xt_sort
...@@ -149,10 +151,14 @@ $(XT_CONFIG_F_MOD_NAME).$(FCMODEXT): ../../src/$(XT_CONFIG_F_MOD_NAME).$(FCMODEX ...@@ -149,10 +151,14 @@ $(XT_CONFIG_F_MOD_NAME).$(FCMODEXT): ../../src/$(XT_CONFIG_F_MOD_NAME).$(FCMODEX
$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT): ../../src/$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT) $(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT): ../../src/$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT)
$(LN_S) ../../src/$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT) . $(LN_S) ../../src/$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT) .
$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT): ../../src/$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT)
$(LN_S) ../../src/$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT) .
##MODULE_RECIPE_MARKER do not remove ## ##MODULE_RECIPE_MARKER do not remove ##
##INCLUDE_HEADER_MARKER do not remove ## ##INCLUDE_HEADER_MARKER do not remove ##
include_HEADERS= \ include_HEADERS= \
$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT) \
$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT) \ $(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT) \
$(XT_CONFIG_F_MOD_NAME).$(FCMODEXT) \ $(XT_CONFIG_F_MOD_NAME).$(FCMODEXT) \
$(XT_SORT_MOD_NAME).$(FCMODEXT) \ $(XT_SORT_MOD_NAME).$(FCMODEXT) \
......
...@@ -88,7 +88,7 @@ void xt_config_set_exchange_method(Xt_config config, int method); ...@@ -88,7 +88,7 @@ void xt_config_set_exchange_method(Xt_config config, int method);
/** /**
* get exchanger used when the \a config object is passed to constructors * get exchanger used when the \a config object is passed to constructors
* @param[in] config configuration object to modify * @param[in] config configuration object to query
* @return an entry from \a Xt_exchangers representing the method of * @return an entry from \a Xt_exchangers representing the method of
* data transfer used * data transfer used
*/ */
...@@ -105,6 +105,32 @@ int xt_config_get_exchange_method(Xt_config config); ...@@ -105,6 +105,32 @@ int xt_config_get_exchange_method(Xt_config config);
int int
xt_exchanger_id_by_name(const char *name); xt_exchanger_id_by_name(const char *name);
/**
* query size at which index lists of vector type will be converted
*
* For many operations it makes sense to first compress large index
* vectors to stripes before continuing further computations.
*
* @param[in] config configuration object to query
* @return size of vectors at which conversion happens
*/
int
xt_config_get_idxvec_autoconvert_size(Xt_config config);
/**
* query size at which index lists of vector type will be converted
*
* For many operations it makes sense to first compress large index
* vectors to stripes before continuing further computations. This
* function sets the size of vectors at which this conversion
* happens for operations that are called with the configuration
* object as parameter.
*
* @param[in,out] config configuration object to modify
* @param[in] cnvsize size of vectors at which conversion happens
*/
void
xt_config_set_idxvec_autoconvert_size(Xt_config config, int cnvsize);
#endif #endif
......
...@@ -53,6 +53,7 @@ ...@@ -53,6 +53,7 @@
#endif #endif
#include "xt/xt_core.h" #include "xt/xt_core.h"
#include "xt/xt_config.h"
/** \example test_xmap_dist_dir.c /** \example test_xmap_dist_dir.c
*/ */
...@@ -80,6 +81,24 @@ ...@@ -80,6 +81,24 @@
Xt_xmap xt_xmap_dist_dir_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, Xt_xmap xt_xmap_dist_dir_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm); MPI_Comm comm);
/**
* Construct an exchange map.\n
* This operation is collective over all processes in comm. \n
* It uses a distributed directory to reduce communication and
* computation during the initialisation at the cost of some extra
* latency because more network transfers than for @a xt_xmap_all2all_new
* are required.
*
* @param[in] src_idxlist source index list
* @param[in] dst_idxlist destination index list
* @param[in] comm MPI communicator that contains all processes
* that take part in the exchange (xt_xmap_dist_dir_new
* will make its own copy of comm)
*/
Xt_xmap
xt_xmap_dist_dir_custom_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm, Xt_config config);
/** /**
* Construct an exchange map.\n * Construct an exchange map.\n
* This operation is collective over all processes in comm. \n * This operation is collective over all processes in comm. \n
...@@ -99,6 +118,28 @@ Xt_xmap ...@@ -99,6 +118,28 @@ Xt_xmap
xt_xmap_dist_dir_intracomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, xt_xmap_dist_dir_intracomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm); MPI_Comm comm);
/**
* Construct an exchange map.\n
* This operation is collective over all processes in comm. \n
* It uses a distributed directory to reduce communication and
* computation during the initialisation at the cost of some extra
* latency because more network transfers than for \a xt_xmap_all2all_new
* are required.
*
* @param[in] src_idxlist source index list
* @param[in] dst_idxlist destination index list
* @param[in] comm MPI communicator that contains all processes
* that take part in the exchange (xt_xmap_dist_dir_new
* will make its own copy of comm), must be an
* intracommunicator.
* @param[in] config custom configuration parameters
*/
Xt_xmap
xt_xmap_dist_dir_intracomm_custom_new(Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
MPI_Comm comm,
Xt_config config);
#endif // XT_XMAP_DIST_DIR_H #endif // XT_XMAP_DIST_DIR_H
/* /*
......
...@@ -54,6 +54,7 @@ ...@@ -54,6 +54,7 @@
#include <stdbool.h> #include <stdbool.h>
#include "xt/xt_config.h"
#include "xt/xt_core.h" #include "xt/xt_core.h"
/** \example test_xmap_dist_dir_intercomm_parallel.c /** \example test_xmap_dist_dir_intercomm_parallel.c
...@@ -78,6 +79,26 @@ Xt_xmap ...@@ -78,6 +79,26 @@ Xt_xmap
xt_xmap_dist_dir_intercomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, xt_xmap_dist_dir_intercomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm inter_comm, MPI_Comm intra_comm); MPI_Comm inter_comm, MPI_Comm intra_comm);
/**
* Construct an exchange map over an intercommunicator with custom parameters\n
* This operation is collective over all processes in comm. \n
* It uses a distributed directory for the initialisation
*
* @param[in] src_idxlist source index list
* @param[in] dst_idxlist destination index list
* @param[in] inter_comm MPI intercommunicator that contains all processes
* that part in the exchange (xt_xmap_dist_dir_icomm_new
* will make its own copy of comm)
* @param[in] intra_comm MPI intracommunicator corresponding to the group
* of @a inter_comm that the calling rank belongs to
* @param[in] config custom parameter object
*/
Xt_xmap
xt_xmap_dist_dir_intercomm_custom_new(Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
MPI_Comm inter_comm, MPI_Comm intra_comm,
Xt_config config);
#endif // XT_XMAP_DIST_DIR_INTERCOMM_H #endif // XT_XMAP_DIST_DIR_INTERCOMM_H
/* /*
......
...@@ -48,6 +48,7 @@ ...@@ -48,6 +48,7 @@
#include <config.h> #include <config.h>
#endif #endif
#include <errno.h>
#include <string.h> #include <string.h>
#include <mpi.h> #include <mpi.h>
...@@ -59,11 +60,13 @@ ...@@ -59,11 +60,13 @@
#include "xt_exchanger_mix_isend_irecv.h" #include "xt_exchanger_mix_isend_irecv.h"
#include "xt_exchanger_irecv_isend_packed.h" #include "xt_exchanger_irecv_isend_packed.h"
#include "xt_exchanger_neigh_alltoall.h" #include "xt_exchanger_neigh_alltoall.h"
#include "xt_idxlist_internal.h"
#include "core/core.h" #include "core/core.h"
#include "core/ppm_xfuncs.h" #include "core/ppm_xfuncs.h"
struct Xt_config_ xt_default_config = { struct Xt_config_ xt_default_config = {
.exchanger_new = xt_exchanger_mix_isend_irecv_new, .exchanger_new = xt_exchanger_mix_isend_irecv_new,
.idxv_cnv_size = CHEAP_VECTOR_SIZE,
}; };
Xt_config xt_config_new(void) Xt_config xt_config_new(void)
...@@ -167,6 +170,20 @@ abort: ...@@ -167,6 +170,20 @@ abort:
Xt_abort(Xt_default_comm, msg, "xt_config.c", __LINE__); Xt_abort(Xt_default_comm, msg, "xt_config.c", __LINE__);
} }
int xt_config_get_idxvec_autoconvert_size(Xt_config config)
{
return config->idxv_cnv_size;
}
void
xt_config_set_idxvec_autoconvert_size(Xt_config config, int cnvsize)
{
if (cnvsize > 3)
config->idxv_cnv_size = cnvsize;
}
void void
xt_config_defaults_init(void) xt_config_defaults_init(void)
{ {
...@@ -179,6 +196,25 @@ xt_config_defaults_init(void) ...@@ -179,6 +196,25 @@ xt_config_defaults_init(void)
fprintf(stderr, "warning: Unexpected value " fprintf(stderr, "warning: Unexpected value "
"for XT_CONFIG_DEFAULT_EXCHANGE_METHOD=%s\n", config_env); "for XT_CONFIG_DEFAULT_EXCHANGE_METHOD=%s\n", config_env);
} }
config_env = getenv("XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE");
if (config_env) {
char *endptr;
long v = strtol(config_env, &endptr, 0);
if ((errno == ERANGE && (v == LONG_MAX || v == LONG_MIN))
|| (errno != 0 && v == 0)) {
perror("failed to parse value of "
"XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE environment variable");
} else if (endptr == config_env) {
fputs("malformed value of XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE"
" environment variable, no digits were found\n",
stderr);
} else if (v < 1 || v > INT_MAX) {
fprintf(stderr, "value of XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE"
" environment variable (%ld) out of range [1,%d]\n",
v, INT_MAX);
} else
xt_config_set_idxvec_autoconvert_size(&xt_default_config, (int)v);
}
} }
/* /*
......
...@@ -48,6 +48,7 @@ ...@@ -48,6 +48,7 @@
MODULE xt_config_f MODULE xt_config_f
USE, INTRINSIC :: ISO_C_BINDING, ONLY: c_ptr, c_null_ptr, c_int, & USE, INTRINSIC :: ISO_C_BINDING, ONLY: c_ptr, c_null_ptr, c_int, &
c_char, c_null_char c_char, c_null_char
USE xt_core, ONLY: xt_abort
IMPLICIT NONE IMPLICIT NONE
PRIVATE PRIVATE
! note: this type must not be extended to contain any other ! note: this type must not be extended to contain any other
...@@ -75,6 +76,8 @@ MODULE xt_config_f ...@@ -75,6 +76,8 @@ MODULE xt_config_f
PUBLIC :: xt_config_f2c PUBLIC :: xt_config_f2c
PUBLIC :: xt_exchanger_id_by_name PUBLIC :: xt_exchanger_id_by_name
PUBLIC :: xt_config_get_exchange_method, xt_config_set_exchange_method PUBLIC :: xt_config_get_exchange_method, xt_config_set_exchange_method
PUBLIC :: xt_config_get_idxvec_autoconvert_size, &
xt_config_set_idxvec_autoconvert_size
INTEGER, PUBLIC, PARAMETER :: & INTEGER, PUBLIC, PARAMETER :: &
xt_exchanger_irecv_send = 0, & xt_exchanger_irecv_send = 0, &
xt_exchanger_irecv_isend = 1, & xt_exchanger_irecv_isend = 1, &
...@@ -82,6 +85,8 @@ MODULE xt_config_f ...@@ -82,6 +85,8 @@ MODULE xt_config_f
xt_exchanger_mix_isend_irecv = 3, & xt_exchanger_mix_isend_irecv = 3, &
xt_exchanger_neigh_alltoall = 4 xt_exchanger_neigh_alltoall = 4
CHARACTER(len=*), PARAMETER :: filename = 'xt_config_f.f90'
CONTAINS CONTAINS
FUNCTION xt_config_new() RESULT(config) FUNCTION xt_config_new() RESULT(config)
...@@ -162,6 +167,39 @@ CONTAINS ...@@ -162,6 +167,39 @@ CONTAINS
exchanger_id = INT(c_id) exchanger_id = INT(c_id)
END FUNCTION xt_exchanger_id_by_name END FUNCTION xt_exchanger_id_by_name
FUNCTION xt_config_get_idxvec_autoconvert_size(config) RESULT(cnvsize)
TYPE(xt_config), INTENT(in) :: config
INTEGER :: cnvsize
INTERFACE
FUNCTION xt_config_get_idxvec_autoconvert_size_c(config) RESULT(cnvsize) &
BIND(c, name='xt_config_get_idxvec_autoconvert_size')
IMPORT :: c_int, c_ptr
TYPE(c_ptr), VALUE :: config
INTEGER(c_int) :: cnvsize
END FUNCTION xt_config_get_idxvec_autoconvert_size_c
END INTERFACE
cnvsize = INT(xt_config_get_idxvec_autoconvert_size_c(config%cptr))
END FUNCTION xt_config_get_idxvec_autoconvert_size
SUBROUTINE xt_config_set_idxvec_autoconvert_size(config, cnvsize)
TYPE(xt_config), INTENT(inout) :: config
INTEGER, INTENT(in) :: cnvsize
INTEGER(c_int) :: cnvsize_c
INTERFACE
SUBROUTINE xt_config_set_idxvec_autoconvert_size_c(config, cnvsize) &
BIND(c, name='xt_config_set_idxvec_autoconvert_size')
IMPORT :: c_int, c_ptr
TYPE(c_ptr), VALUE :: config
INTEGER(c_int), VALUE :: cnvsize
END SUBROUTINE xt_config_set_idxvec_autoconvert_size_c
END INTERFACE
IF (cnvsize > HUGE(1_c_int) .OR. cnvsize < 1) &
CALL xt_abort("invalid conversion size", filename, __LINE__)
cnvsize_c = INT(cnvsize, c_int)
CALL xt_config_set_idxvec_autoconvert_size_c(config%cptr, cnvsize_c)
END SUBROUTINE xt_config_set_idxvec_autoconvert_size
END MODULE xt_config_f END MODULE xt_config_f
! !
! Local Variables: ! Local Variables:
......
...@@ -55,7 +55,15 @@ ...@@ -55,7 +55,15 @@
#include "xt_exchanger.h" #include "xt_exchanger.h"
struct Xt_config_ { struct Xt_config_ {
/**
* constructor to use when creating the exchanger of a redist
*/
Xt_exchanger_new exchanger_new; Xt_exchanger_new exchanger_new;
/**
* automatically compress index lists of vector type at this size
* into another representation to save on computation/memory overall
*/
int idxv_cnv_size;
}; };
extern struct Xt_config_ xt_default_config; extern struct Xt_config_ xt_default_config;
......
...@@ -64,6 +64,7 @@ ...@@ -64,6 +64,7 @@
#include "core/ppm_xfuncs.h" #include "core/ppm_xfuncs.h"
#include "xt/xt_xmap_all2all.h" #include "xt/xt_xmap_all2all.h"
#include "xt/xt_xmap_intersection.h" #include "xt/xt_xmap_intersection.h"
#include "xt_config_internal.h"
#include "xt_idxlist_internal.h" #include "xt_idxlist_internal.h"
#include "instr.h" #include "instr.h"
...@@ -74,7 +75,7 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections, ...@@ -74,7 +75,7 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections,
int * stripify, int * stripify,
Xt_idxlist src_idxlist_local, Xt_idxlist src_idxlist_local,
Xt_idxlist dst_idxlist_local, Xt_idxlist dst_idxlist_local,
MPI_Comm comm) MPI_Comm comm, Xt_config config)
{ {
/* /*
...@@ -158,8 +159,8 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections, ...@@ -158,8 +159,8 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections,
dst = dst_idxlist_local; dst = dst_idxlist_local;
} }
large_list_seen large_list_seen
|= (xt_idxlist_get_num_indices(src) >= CHEAP_VECTOR_SIZE) |= (xt_idxlist_get_num_indices(src) >= config->idxv_cnv_size)
| (xt_idxlist_get_num_indices(dst) >= CHEAP_VECTOR_SIZE); | (xt_idxlist_get_num_indices(dst) >= config->idxv_cnv_size);
Xt_idxlist intersect = xt_idxlist_get_intersection(src, dst_idxlist_local); Xt_idxlist intersect = xt_idxlist_get_intersection(src, dst_idxlist_local);
if (xt_idxlist_get_num_indices(intersect) > 0) { if (xt_idxlist_get_num_indices(intersect) > 0) {
...@@ -200,7 +201,9 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections, ...@@ -200,7 +201,9 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections,
*dst_intersections = dsti; *dst_intersections = dsti;
} }
Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_Comm comm) { Xt_xmap
xt_xmap_all2all_custom_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm, Xt_config config) {
INSTR_DEF(t_xt_xmap_all2all_new,"xt_xmap_all2all_new") INSTR_DEF(t_xt_xmap_all2all_new,"xt_xmap_all2all_new")
INSTR_START(t_xt_xmap_all2all_new); INSTR_START(t_xt_xmap_all2all_new);
...@@ -217,7 +220,8 @@ Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_ ...@@ -217,7 +220,8 @@ Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_
// exchange index lists between all processes in comm // exchange index lists between all processes in comm
exchange_idxlists(&src_intersections, &num_src_intersections, exchange_idxlists(&src_intersections, &num_src_intersections,
&dst_intersections, &num_dst_intersections, &dst_intersections, &num_dst_intersections,
&stripify, src_idxlist, dst_idxlist, newcomm); &stripify, src_idxlist, dst_idxlist, newcomm,
config);
Xt_xmap (*xmap_new)(int num_src_intersections, Xt_xmap (*xmap_new)(int num_src_intersections,
const struct Xt_com_list *src_com, const struct Xt_com_list *src_com,
...@@ -244,6 +248,14 @@ Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_ ...@@ -244,6 +248,14 @@ Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_
return xmap; return xmap;
} }
Xt_xmap
xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm)
{
return xt_xmap_all2all_custom_new(src_idxlist, dst_idxlist,
comm, &xt_default_config);
}
/* /*
* Local Variables: * Local Variables:
* c-basic-offset: 2 * c-basic-offset: 2
......
...@@ -68,20 +68,22 @@ ...@@ -68,20 +68,22 @@
#include "core/core.h" #include "core/core.h"
#include "core/ppm_xfuncs.h" #include "core/ppm_xfuncs.h"
#include "xt/xt_xmap_intersection.h" #include "xt/xt_xmap_intersection.h"
#include "xt_config_internal.h"
#include "xt_idxlist_internal.h" #include "xt_idxlist_internal.h"
#include "instr.h" #include "instr.h"
#include "xt_xmap_dist_dir_common.h" #include "xt_xmap_dist_dir_common.h"
static inline Xt_int static inline Xt_int
get_dist_dir_global_interval_size(Xt_idxlist src, Xt_idxlist dst, get_dist_dir_global_interval_size(Xt_idxlist src, Xt_idxlist dst,
bool *stripify, MPI_Comm comm, int comm_size) bool *stripify, MPI_Comm comm, int comm_size,
Xt_config config)
{ {
unsigned long long local_vals[2], global_sums[2]; unsigned long long local_vals[2], global_sums[2];
unsigned num_indices_src = (unsigned)xt_idxlist_get_num_indices(src); unsigned num_indices_src = (unsigned)xt_idxlist_get_num_indices(src);
local_vals[0] = num_indices_src; local_vals[0] = num_indices_src;
local_vals[1] = (num_indices_src >= CHEAP_VECTOR_SIZE) local_vals[1] = (num_indices_src >= (unsigned)config->idxv_cnv_size)
|| (xt_idxlist_get_num_indices(dst) >= CHEAP_VECTOR_SIZE); || (xt_idxlist_get_num_indices(dst) >= config->idxv_cnv_size);
xt_mpi_call(MPI_Allreduce(local_vals, global_sums, 2, xt_mpi_call(MPI_Allreduce(local_vals, global_sums, 2,
MPI_UNSIGNED_LONG_LONG, MPI_SUM, comm), comm); MPI_UNSIGNED_LONG_LONG, MPI_SUM, comm), comm);
...@@ -113,13 +115,14 @@ static inline Xt_int get_max_idxlist_index(Xt_idxlist a, Xt_idxlist b) { ...@@ -113,13 +115,14 @@ static inline Xt_int get_max_idxlist_index(Xt_idxlist a, Xt_idxlist b) {
static struct bucket_params static struct bucket_params
get_bucket_params(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, get_bucket_params(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
bool *stripify, MPI_Comm comm, int comm_size) bool *stripify, MPI_Comm comm, int comm_size,
Xt_config config)
{ {
/* global_interval is a multiple of comm_size and has a size of at least /* global_interval is a multiple of comm_size and has a size of at least
comm_size */ comm_size */
Xt_int global_interval Xt_int global_interval
= get_dist_dir_global_interval_size(src_idxlist, dst_idxlist, stripify, = get_dist_dir_global_interval_size(src_idxlist, dst_idxlist, stripify,
comm, comm_size); comm, comm_size, config);
Xt_int local_interval = (Xt_int)(global_interval / comm_size); Xt_int local_interval = (Xt_int)(global_interval / comm_size);
Xt_int local_index_range_lbound Xt_int local_index_range_lbound
= get_min_idxlist_index(src_idxlist, dst_idxlist); = get_min_idxlist_index(src_idxlist, dst_idxlist);
...@@ -452,10 +455,12 @@ static void generate_distributed_directories(struct dist_dir **src_dist_dir, ...@@ -452,10 +455,12 @@ static void generate_distributed_directories(struct dist_dir **src_dist_dir,
Xt_idxlist src_idxlist, Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist, Xt_idxlist dst_idxlist,
int tag_offset, int tag_offset,
MPI_Comm comm, int comm_size) { MPI_Comm comm, int comm_size,
Xt_config config) {
struct bucket_params bucket_params struct bucket_params bucket_params
= get_bucket_params(src_idxlist, dst_idxlist, stripify, comm, comm_size); = get_bucket_params(src_idxlist, dst_idxlist, stripify, comm, comm_size,
config);
void *send_buffer = NULL; void *send_buffer = NULL;
...@@ -606,7 +611,8 @@ static void exchange_idxlists(struct dist_dir **src_intersections, ...@@ -606,7 +611,8 @@ static void exchange_idxlists(struct dist_dir **src_intersections,
Xt_idxlist src_idxlist, Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist, Xt_idxlist dst_idxlist,
int tag_offset, int tag_offset,
MPI_Comm comm) { MPI_Comm comm,
Xt_config config) {
int comm_size; int comm_size;
...@@ -616,7 +622,7 @@ static void exchange_idxlists(struct dist_dir **src_intersections, ...@@ -616,7 +622,7 @@ static void exchange_idxlists(struct dist_dir **src_intersections,
generate_distributed_directories(&src_dist_dir, &dst_dist_dir, stripify, generate_distributed_directories(&src_dist_dir, &dst_dist_dir, stripify,
src_idxlist, dst_idxlist, src_idxlist, dst_idxlist,
tag_offset, comm, comm_size); tag_offset, comm, comm_size