Commit 2d8db34d authored by Thomas Jahns's avatar Thomas Jahns 🤸
Browse files

Make threshold for automatic vector to stripes conversion tunable.

parent 0d657ee9
......@@ -35,6 +35,7 @@
if FC_MOD_UPPERCASE
##UPPERCASE_MODULE_NAME_MARKER do not remove##
XT_XMAP_RENAME_MOD_NAME=XT_XMAP_RENAME
XT_REDIST_RENAME_MOD_NAME=XT_REDIST_RENAME
XT_CONFIG_F_MOD_NAME=XT_CONFIG_F
XT_SORT_MOD_NAME=XT_SORT
......@@ -59,6 +60,7 @@ XT_UT_MOD_NAME=XT_UT
YAXT_MOD_NAME=YAXT
else
##LOWERCASE_MODULE_NAME_MARKER do not remove##
XT_XMAP_RENAME_MOD_NAME=xt_xmap_rename
XT_REDIST_RENAME_MOD_NAME=xt_redist_rename
XT_CONFIG_F_MOD_NAME=xt_config_f
XT_SORT_MOD_NAME=xt_sort
......@@ -149,10 +151,14 @@ $(XT_CONFIG_F_MOD_NAME).$(FCMODEXT): ../../src/$(XT_CONFIG_F_MOD_NAME).$(FCMODEX
$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT): ../../src/$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT)
$(LN_S) ../../src/$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT) .
$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT): ../../src/$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT)
$(LN_S) ../../src/$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT) .
##MODULE_RECIPE_MARKER do not remove ##
##INCLUDE_HEADER_MARKER do not remove ##
include_HEADERS= \
$(XT_XMAP_RENAME_MOD_NAME).$(FCMODEXT) \
$(XT_REDIST_RENAME_MOD_NAME).$(FCMODEXT) \
$(XT_CONFIG_F_MOD_NAME).$(FCMODEXT) \
$(XT_SORT_MOD_NAME).$(FCMODEXT) \
......
......@@ -88,7 +88,7 @@ void xt_config_set_exchange_method(Xt_config config, int method);
/**
* get exchanger used when the \a config object is passed to constructors
* @param[in] config configuration object to modify
* @param[in] config configuration object to query
* @return an entry from \a Xt_exchangers representing the method of
* data transfer used
*/
......@@ -105,6 +105,32 @@ int xt_config_get_exchange_method(Xt_config config);
int
xt_exchanger_id_by_name(const char *name);
/**
* query size at which index lists of vector type will be converted
*
* For many operations it makes sense to first compress large index
* vectors to stripes before continuing further computations.
*
* @param[in] config configuration object to query
* @return size of vectors at which conversion happens
*/
int
xt_config_get_idxvec_autoconvert_size(Xt_config config);
/**
* query size at which index lists of vector type will be converted
*
* For many operations it makes sense to first compress large index
* vectors to stripes before continuing further computations. This
* function sets the size of vectors at which this conversion
* happens for operations that are called with the configuration
* object as parameter.
*
* @param[in,out] config configuration object to modify
* @param[in] cnvsize size of vectors at which conversion happens
*/
void
xt_config_set_idxvec_autoconvert_size(Xt_config config, int cnvsize);
#endif
......
......@@ -53,6 +53,7 @@
#endif
#include "xt/xt_core.h"
#include "xt/xt_config.h"
/** \example test_xmap_dist_dir.c
*/
......@@ -80,6 +81,24 @@
Xt_xmap xt_xmap_dist_dir_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm);
/**
* Construct an exchange map.\n
* This operation is collective over all processes in comm. \n
* It uses a distributed directory to reduce communication and
* computation during the initialisation at the cost of some extra
* latency because more network transfers than for @a xt_xmap_all2all_new
* are required.
*
* @param[in] src_idxlist source index list
* @param[in] dst_idxlist destination index list
* @param[in] comm MPI communicator that contains all processes
* that take part in the exchange (xt_xmap_dist_dir_new
* will make its own copy of comm)
*/
Xt_xmap
xt_xmap_dist_dir_custom_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm, Xt_config config);
/**
* Construct an exchange map.\n
* This operation is collective over all processes in comm. \n
......@@ -99,6 +118,28 @@ Xt_xmap
xt_xmap_dist_dir_intracomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm);
/**
* Construct an exchange map.\n
* This operation is collective over all processes in comm. \n
* It uses a distributed directory to reduce communication and
* computation during the initialisation at the cost of some extra
* latency because more network transfers than for \a xt_xmap_all2all_new
* are required.
*
* @param[in] src_idxlist source index list
* @param[in] dst_idxlist destination index list
* @param[in] comm MPI communicator that contains all processes
* that take part in the exchange (xt_xmap_dist_dir_new
* will make its own copy of comm), must be an
* intracommunicator.
* @param[in] config custom configuration parameters
*/
Xt_xmap
xt_xmap_dist_dir_intracomm_custom_new(Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
MPI_Comm comm,
Xt_config config);
#endif // XT_XMAP_DIST_DIR_H
/*
......
......@@ -54,6 +54,7 @@
#include <stdbool.h>
#include "xt/xt_config.h"
#include "xt/xt_core.h"
/** \example test_xmap_dist_dir_intercomm_parallel.c
......@@ -78,6 +79,26 @@ Xt_xmap
xt_xmap_dist_dir_intercomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm inter_comm, MPI_Comm intra_comm);
/**
* Construct an exchange map over an intercommunicator with custom parameters\n
* This operation is collective over all processes in comm. \n
* It uses a distributed directory for the initialisation
*
* @param[in] src_idxlist source index list
* @param[in] dst_idxlist destination index list
* @param[in] inter_comm MPI intercommunicator that contains all processes
* that part in the exchange (xt_xmap_dist_dir_icomm_new
* will make its own copy of comm)
* @param[in] intra_comm MPI intracommunicator corresponding to the group
* of @a inter_comm that the calling rank belongs to
* @param[in] config custom parameter object
*/
Xt_xmap
xt_xmap_dist_dir_intercomm_custom_new(Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
MPI_Comm inter_comm, MPI_Comm intra_comm,
Xt_config config);
#endif // XT_XMAP_DIST_DIR_INTERCOMM_H
/*
......
......@@ -48,6 +48,7 @@
#include <config.h>
#endif
#include <errno.h>
#include <string.h>
#include <mpi.h>
......@@ -59,11 +60,13 @@
#include "xt_exchanger_mix_isend_irecv.h"
#include "xt_exchanger_irecv_isend_packed.h"
#include "xt_exchanger_neigh_alltoall.h"
#include "xt_idxlist_internal.h"
#include "core/core.h"
#include "core/ppm_xfuncs.h"
struct Xt_config_ xt_default_config = {
.exchanger_new = xt_exchanger_mix_isend_irecv_new,
.idxv_cnv_size = CHEAP_VECTOR_SIZE,
};
Xt_config xt_config_new(void)
......@@ -167,6 +170,20 @@ abort:
Xt_abort(Xt_default_comm, msg, "xt_config.c", __LINE__);
}
int xt_config_get_idxvec_autoconvert_size(Xt_config config)
{
return config->idxv_cnv_size;
}
void
xt_config_set_idxvec_autoconvert_size(Xt_config config, int cnvsize)
{
if (cnvsize > 3)
config->idxv_cnv_size = cnvsize;
}
void
xt_config_defaults_init(void)
{
......@@ -179,6 +196,25 @@ xt_config_defaults_init(void)
fprintf(stderr, "warning: Unexpected value "
"for XT_CONFIG_DEFAULT_EXCHANGE_METHOD=%s\n", config_env);
}
config_env = getenv("XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE");
if (config_env) {
char *endptr;
long v = strtol(config_env, &endptr, 0);
if ((errno == ERANGE && (v == LONG_MAX || v == LONG_MIN))
|| (errno != 0 && v == 0)) {
perror("failed to parse value of "
"XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE environment variable");
} else if (endptr == config_env) {
fputs("malformed value of XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE"
" environment variable, no digits were found\n",
stderr);
} else if (v < 1 || v > INT_MAX) {
fprintf(stderr, "value of XT_CONFIG_DEFAULT_IDXVEC_AUTOCONVERT_SIZE"
" environment variable (%ld) out of range [1,%d]\n",
v, INT_MAX);
} else
xt_config_set_idxvec_autoconvert_size(&xt_default_config, (int)v);
}
}
/*
......
......@@ -48,6 +48,7 @@
MODULE xt_config_f
USE, INTRINSIC :: ISO_C_BINDING, ONLY: c_ptr, c_null_ptr, c_int, &
c_char, c_null_char
USE xt_core, ONLY: xt_abort
IMPLICIT NONE
PRIVATE
! note: this type must not be extended to contain any other
......@@ -75,6 +76,8 @@ MODULE xt_config_f
PUBLIC :: xt_config_f2c
PUBLIC :: xt_exchanger_id_by_name
PUBLIC :: xt_config_get_exchange_method, xt_config_set_exchange_method
PUBLIC :: xt_config_get_idxvec_autoconvert_size, &
xt_config_set_idxvec_autoconvert_size
INTEGER, PUBLIC, PARAMETER :: &
xt_exchanger_irecv_send = 0, &
xt_exchanger_irecv_isend = 1, &
......@@ -82,6 +85,8 @@ MODULE xt_config_f
xt_exchanger_mix_isend_irecv = 3, &
xt_exchanger_neigh_alltoall = 4
CHARACTER(len=*), PARAMETER :: filename = 'xt_config_f.f90'
CONTAINS
FUNCTION xt_config_new() RESULT(config)
......@@ -162,6 +167,39 @@ CONTAINS
exchanger_id = INT(c_id)
END FUNCTION xt_exchanger_id_by_name
FUNCTION xt_config_get_idxvec_autoconvert_size(config) RESULT(cnvsize)
TYPE(xt_config), INTENT(in) :: config
INTEGER :: cnvsize
INTERFACE
FUNCTION xt_config_get_idxvec_autoconvert_size_c(config) RESULT(cnvsize) &
BIND(c, name='xt_config_get_idxvec_autoconvert_size')
IMPORT :: c_int, c_ptr
TYPE(c_ptr), VALUE :: config
INTEGER(c_int) :: cnvsize
END FUNCTION xt_config_get_idxvec_autoconvert_size_c
END INTERFACE
cnvsize = INT(xt_config_get_idxvec_autoconvert_size_c(config%cptr))
END FUNCTION xt_config_get_idxvec_autoconvert_size
SUBROUTINE xt_config_set_idxvec_autoconvert_size(config, cnvsize)
TYPE(xt_config), INTENT(inout) :: config
INTEGER, INTENT(in) :: cnvsize
INTEGER(c_int) :: cnvsize_c
INTERFACE
SUBROUTINE xt_config_set_idxvec_autoconvert_size_c(config, cnvsize) &
BIND(c, name='xt_config_set_idxvec_autoconvert_size')
IMPORT :: c_int, c_ptr
TYPE(c_ptr), VALUE :: config
INTEGER(c_int), VALUE :: cnvsize
END SUBROUTINE xt_config_set_idxvec_autoconvert_size_c
END INTERFACE
IF (cnvsize > HUGE(1_c_int) .OR. cnvsize < 1) &
CALL xt_abort("invalid conversion size", filename, __LINE__)
cnvsize_c = INT(cnvsize, c_int)
CALL xt_config_set_idxvec_autoconvert_size_c(config%cptr, cnvsize_c)
END SUBROUTINE xt_config_set_idxvec_autoconvert_size
END MODULE xt_config_f
!
! Local Variables:
......
......@@ -55,7 +55,15 @@
#include "xt_exchanger.h"
struct Xt_config_ {
/**
* constructor to use when creating the exchanger of a redist
*/
Xt_exchanger_new exchanger_new;
/**
* automatically compress index lists of vector type at this size
* into another representation to save on computation/memory overall
*/
int idxv_cnv_size;
};
extern struct Xt_config_ xt_default_config;
......
......@@ -64,6 +64,7 @@
#include "core/ppm_xfuncs.h"
#include "xt/xt_xmap_all2all.h"
#include "xt/xt_xmap_intersection.h"
#include "xt_config_internal.h"
#include "xt_idxlist_internal.h"
#include "instr.h"
......@@ -74,7 +75,7 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections,
int * stripify,
Xt_idxlist src_idxlist_local,
Xt_idxlist dst_idxlist_local,
MPI_Comm comm)
MPI_Comm comm, Xt_config config)
{
/*
......@@ -158,8 +159,8 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections,
dst = dst_idxlist_local;
}
large_list_seen
|= (xt_idxlist_get_num_indices(src) >= CHEAP_VECTOR_SIZE)
| (xt_idxlist_get_num_indices(dst) >= CHEAP_VECTOR_SIZE);
|= (xt_idxlist_get_num_indices(src) >= config->idxv_cnv_size)
| (xt_idxlist_get_num_indices(dst) >= config->idxv_cnv_size);
Xt_idxlist intersect = xt_idxlist_get_intersection(src, dst_idxlist_local);
if (xt_idxlist_get_num_indices(intersect) > 0) {
......@@ -200,7 +201,9 @@ static void exchange_idxlists(struct Xt_com_list **src_intersections,
*dst_intersections = dsti;
}
Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_Comm comm) {
Xt_xmap
xt_xmap_all2all_custom_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm, Xt_config config) {
INSTR_DEF(t_xt_xmap_all2all_new,"xt_xmap_all2all_new")
INSTR_START(t_xt_xmap_all2all_new);
......@@ -217,7 +220,8 @@ Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_
// exchange index lists between all processes in comm
exchange_idxlists(&src_intersections, &num_src_intersections,
&dst_intersections, &num_dst_intersections,
&stripify, src_idxlist, dst_idxlist, newcomm);
&stripify, src_idxlist, dst_idxlist, newcomm,
config);
Xt_xmap (*xmap_new)(int num_src_intersections,
const struct Xt_com_list *src_com,
......@@ -244,6 +248,14 @@ Xt_xmap xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist, MPI_
return xmap;
}
Xt_xmap
xt_xmap_all2all_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm)
{
return xt_xmap_all2all_custom_new(src_idxlist, dst_idxlist,
comm, &xt_default_config);
}
/*
* Local Variables:
* c-basic-offset: 2
......
......@@ -68,20 +68,22 @@
#include "core/core.h"
#include "core/ppm_xfuncs.h"
#include "xt/xt_xmap_intersection.h"
#include "xt_config_internal.h"
#include "xt_idxlist_internal.h"
#include "instr.h"
#include "xt_xmap_dist_dir_common.h"
static inline Xt_int
get_dist_dir_global_interval_size(Xt_idxlist src, Xt_idxlist dst,
bool *stripify, MPI_Comm comm, int comm_size)
bool *stripify, MPI_Comm comm, int comm_size,
Xt_config config)
{
unsigned long long local_vals[2], global_sums[2];
unsigned num_indices_src = (unsigned)xt_idxlist_get_num_indices(src);
local_vals[0] = num_indices_src;
local_vals[1] = (num_indices_src >= CHEAP_VECTOR_SIZE)
|| (xt_idxlist_get_num_indices(dst) >= CHEAP_VECTOR_SIZE);
local_vals[1] = (num_indices_src >= (unsigned)config->idxv_cnv_size)
|| (xt_idxlist_get_num_indices(dst) >= config->idxv_cnv_size);
xt_mpi_call(MPI_Allreduce(local_vals, global_sums, 2,
MPI_UNSIGNED_LONG_LONG, MPI_SUM, comm), comm);
......@@ -113,13 +115,14 @@ static inline Xt_int get_max_idxlist_index(Xt_idxlist a, Xt_idxlist b) {
static struct bucket_params
get_bucket_params(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
bool *stripify, MPI_Comm comm, int comm_size)
bool *stripify, MPI_Comm comm, int comm_size,
Xt_config config)
{
/* global_interval is a multiple of comm_size and has a size of at least
comm_size */
Xt_int global_interval
= get_dist_dir_global_interval_size(src_idxlist, dst_idxlist, stripify,
comm, comm_size);
comm, comm_size, config);
Xt_int local_interval = (Xt_int)(global_interval / comm_size);
Xt_int local_index_range_lbound
= get_min_idxlist_index(src_idxlist, dst_idxlist);
......@@ -452,10 +455,12 @@ static void generate_distributed_directories(struct dist_dir **src_dist_dir,
Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
int tag_offset,
MPI_Comm comm, int comm_size) {
MPI_Comm comm, int comm_size,
Xt_config config) {
struct bucket_params bucket_params
= get_bucket_params(src_idxlist, dst_idxlist, stripify, comm, comm_size);
= get_bucket_params(src_idxlist, dst_idxlist, stripify, comm, comm_size,
config);
void *send_buffer = NULL;
......@@ -606,7 +611,8 @@ static void exchange_idxlists(struct dist_dir **src_intersections,
Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
int tag_offset,
MPI_Comm comm) {
MPI_Comm comm,
Xt_config config) {
int comm_size;
......@@ -616,7 +622,7 @@ static void exchange_idxlists(struct dist_dir **src_intersections,
generate_distributed_directories(&src_dist_dir, &dst_dist_dir, stripify,
src_idxlist, dst_idxlist,
tag_offset, comm, comm_size);
tag_offset, comm, comm_size, config);
void * send_buffer;
......@@ -663,8 +669,9 @@ static void exchange_idxlists(struct dist_dir **src_intersections,
}
Xt_xmap
xt_xmap_dist_dir_intracomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm)
xt_xmap_dist_dir_intracomm_custom_new(
Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm, Xt_config config)
{
INSTR_DEF(this_instr,"xt_xmap_all2all_new")
INSTR_START(this_instr);
......@@ -679,7 +686,7 @@ xt_xmap_dist_dir_intracomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
bool stripify;
exchange_idxlists(&src_intersections, &dst_intersections, &stripify,
src_idxlist, dst_idxlist, tag_offset, newcomm);
src_idxlist, dst_idxlist, tag_offset, newcomm, config);
Xt_xmap (*xmap_new)(int num_src_intersections,
const struct Xt_com_list *src_com,
......@@ -702,6 +709,14 @@ xt_xmap_dist_dir_intracomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
return xmap;
}
Xt_xmap
xt_xmap_dist_dir_intracomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm)
{
return xt_xmap_dist_dir_intracomm_custom_new(src_idxlist, dst_idxlist, comm,
&xt_default_config);
}
/*
* Local Variables:
* c-basic-offset: 2
......
......@@ -62,6 +62,7 @@
#include "xt_xmap_dist_dir_common.h"
#include "ensure_array_size.h"
#include "xt_idxstripes_internal.h"
#include "xt_config_internal.h"
void xt_xmdd_free_dist_dir(struct dist_dir *dist_dir) {
size_t num_entries
......@@ -362,9 +363,17 @@ xt_com_list_rank_cmp(const void *a_, const void *b_)
return a->rank - b->rank;
}
Xt_xmap
xt_xmap_dist_dir_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm)
{
return xt_xmap_dist_dir_custom_new(src_idxlist, dst_idxlist, comm,
&xt_default_config);
}
Xt_xmap xt_xmap_dist_dir_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm)
Xt_xmap
xt_xmap_dist_dir_custom_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm comm, Xt_config config)
{
// ensure that yaxt is initialized
assert(xt_initialized());
......@@ -373,7 +382,8 @@ Xt_xmap xt_xmap_dist_dir_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
xt_mpi_call(MPI_Comm_test_inter(comm, &is_inter), comm);
Xt_xmap xmap;
if (!is_inter)
xmap = xt_xmap_dist_dir_intracomm_new(src_idxlist, dst_idxlist, comm);
xmap = xt_xmap_dist_dir_intracomm_custom_new(src_idxlist, dst_idxlist, comm,
config);
else {
MPI_Comm merge_comm, local_intra_comm;
MPI_Group local_group;
......@@ -385,8 +395,9 @@ Xt_xmap xt_xmap_dist_dir_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
xt_mpi_call(MPI_Comm_free(&merge_comm), comm);
xt_mpi_comm_mark_exclusive(local_intra_comm);
xmap = xt_xmap_dist_dir_intercomm_new(src_idxlist, dst_idxlist,
comm, local_intra_comm);
xmap
= xt_xmap_dist_dir_intercomm_custom_new(src_idxlist, dst_idxlist,
comm, local_intra_comm, config);
xt_mpi_call(MPI_Comm_free(&local_intra_comm), local_intra_comm);
}
......
......@@ -70,10 +70,10 @@
#include "xt_mpi_internal.h"
#include "core/core.h"
#include "core/ppm_xfuncs.h"
#include "ensure_array_size.h"
#include "xt/xt_xmap_intersection.h"
#include "xt_idxlist_internal.h"
#include "xt_xmap_dist_dir_common.h"
#include "xt_config_internal.h"
#include "instr.h"
static inline void
......@@ -81,7 +81,8 @@ get_dist_dir_global_interval_size(Xt_idxlist src, Xt_idxlist dst,
bool *stripify, Xt_int interval_size[2],
MPI_Comm intra_comm, MPI_Comm inter_comm,
int comm_size, int remote_size,
int comm_rank, int tag_offset_inter)
int comm_rank, int tag_offset_inter,
Xt_config config)
{
/* global_sums[0] and [1] refer to the local and remote group of
* intercommunicator inter_comm */
......@@ -89,8 +90,8 @@ get_dist_dir_global_interval_size(Xt_idxlist src, Xt_idxlist dst,
unsigned num_indices_src = (unsigned)xt_idxlist_get_num_indices(src);
local_vals[0] = num_indices_src;
local_vals[1] = (num_indices_src >= CHEAP_VECTOR_SIZE)
|| (xt_idxlist_get_num_indices(dst) >= CHEAP_VECTOR_SIZE);
local_vals[1] = (num_indices_src >= (unsigned)config->idxv_cnv_size)
|| (xt_idxlist_get_num_indices(dst) >= config->idxv_cnv_size);
xt_mpi_call(MPI_Allreduce(local_vals, global_sums[0], 2,
MPI_UNSIGNED_LONG_LONG, MPI_SUM, intra_comm),
......@@ -357,7 +358,7 @@ generate_distributed_directories(struct dist_dir **src_dist_dir,
int tag_offset_inter, int tag_offset_intra,
MPI_Comm inter_comm, MPI_Comm intra_comm,
int remote_size, int comm_size,
int comm_rank) {
int comm_rank, Xt_config config) {
/* interval_size[0] and interval_size[1] are the global interval
* size for the local and remote group */
......@@ -366,7 +367,7 @@ generate_distributed_directories(struct dist_dir **src_dist_dir,
stripify, interval_size,
intra_comm, inter_comm,
comm_size, remote_size,
comm_rank, tag_offset_inter);
comm_rank, tag_offset_inter, config);
void *send_buffer_local, *send_buffer_remote;
int (*send_size_local)[SEND_SIZE_ASIZE]
= xmalloc(((size_t)comm_size + (size_t)remote_size)
......@@ -527,7 +528,8 @@ exchange_idxlists(struct dist_dir **src_intersections,
Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
int tag_offset_inter, int tag_offset_intra,
MPI_Comm inter_comm, MPI_Comm intra_comm) {
MPI_Comm inter_comm, MPI_Comm intra_comm,
Xt_config config) {
int comm_size, remote_size, comm_rank;
xt_mpi_call(MPI_Comm_size(inter_comm, &comm_size), inter_comm);
......@@ -541,7 +543,7 @@ exchange_idxlists(struct dist_dir **src_intersections,
tag_offset_inter, tag_offset_intra,
inter_comm, intra_comm,
remote_size, comm_size,
comm_rank);
comm_rank, config);
int (*send_size_local)[SEND_SIZE_ASIZE]
......@@ -636,8 +638,11 @@ exchange_idxlists(struct dist_dir **src_intersections,
Xt_xmap
xt_xmap_dist_dir_intercomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
MPI_Comm inter_comm_, MPI_Comm intra_comm_)
xt_xmap_dist_dir_intercomm_custom_new(Xt_idxlist src_idxlist,
Xt_idxlist dst_idxlist,
MPI_Comm inter_comm_,
MPI_Comm intra_comm_,
Xt_config config)
{
INSTR_DEF(this_instr,"xt_xmap_dist_dir_intercomm_new")
INSTR_START(this_instr);
......@@ -655,7 +660,7 @@ xt_xmap_dist_dir_intercomm_new(Xt_idxlist src_idxlist, Xt_idxlist dst_idxlist,
exchange_idxlists(&src_intersections, &dst_intersections, &stripify,
src_idxlist, dst_idxlist,
tag_offset_inter, tag_offset_intra,
inter_comm, intra_comm);