xt_redist_collection_static.c 6.15 KB
Newer Older
1
2
3
/**
 * @file xt_redist_collection_static.c
 *
4
5
 * @copyright Copyright  (C)  2016 Jörg Behrens <behrens@dkrz.de>
 *                                 Moritz Hanke <hanke@dkrz.de>
6
7
 *                                 Thomas Jahns <jahns@dkrz.de>
 *
8
9
 * @author Jörg Behrens <behrens@dkrz.de>
 *         Moritz Hanke <hanke@dkrz.de>
10
11
12
13
 *         Thomas Jahns <jahns@dkrz.de>
 */
/*
 * Keywords:
14
15
 * Maintainer: Jörg Behrens <behrens@dkrz.de>
 *             Moritz Hanke <hanke@dkrz.de>
16
 *             Thomas Jahns <jahns@dkrz.de>
Moritz Hanke's avatar
Moritz Hanke committed
17
 * URL: https://doc.redmine.dkrz.de/yaxt/html/
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are  permitted provided that the following conditions are
 * met:
 *
 * Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * Neither the name of the DKRZ GmbH nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
Thomas Jahns's avatar
Thomas Jahns committed
46
47
48
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
49

50
#include <assert.h>
51
52
#include <limits.h>
#include <stdbool.h>
53
54
55
56
57
58
#include <stdlib.h>

#include <mpi.h>

#include "core/core.h"
#include "core/ppm_xfuncs.h"
59
#include "xt/xt_mpi.h"
60
#include "xt_mpi_internal.h"
61
#include "xt/xt_redist_collection_static.h"
62
#include "xt/xt_redist_single_array_base.h"
63
#include "ensure_array_size.h"
64
#include "xt/xt_redist.h"
65
#include "xt_redist_internal.h"
Thomas Jahns's avatar
Thomas Jahns committed
66
#include "xt_config_internal.h"
67
68


Thomas Jahns's avatar
Thomas Jahns committed
69
70
71
72
73
74
75
76
static void
generate_msg_infos(size_t nmsg, size_t num_redists,
                   struct Xt_redist_msg *msgs,
                   const MPI_Aint displacements[num_redists],
                   const Xt_redist redists[num_redists],
                   const size_t num_ranks[num_redists],
                   const int *restrict ranks[num_redists],
                   MPI_Comm comm,
77
78
                   enum xt_msg_direction direction)
{
Thomas Jahns's avatar
Thomas Jahns committed
79
80
81
82
  if (nmsg) {
    size_t rank_pos[num_redists];
    for (size_t j = 0; j < num_redists; ++j)
      rank_pos[j] = 0;
Thomas Jahns's avatar
Thomas Jahns committed
83
84
85
86
    MPI_Datatype datatypes[num_redists];
    int block_lengths[num_redists];
    for (size_t i = 0; i < num_redists; ++i)
      block_lengths[i] = 1;
Thomas Jahns's avatar
Thomas Jahns committed
87
    for (size_t i = 0; i < nmsg; ++i) {
88
89
90
91
92
93
94
95
      int min_rank = INT_MAX;
      for (size_t j = 0; j < num_redists; ++j)
        if (rank_pos[j] < num_ranks[j] && ranks[j][rank_pos[j]] < min_rank)
          min_rank = ranks[j][rank_pos[j]];

      for (size_t j = 0; j < num_redists; ++j)
        datatypes[j] =
          (rank_pos[j] < num_ranks[j] && ranks[j][rank_pos[j]] == min_rank)
96
          ? xt_redist_get_MPI_Datatype(redists[j], min_rank, direction, false)
97
          : MPI_DATATYPE_NULL;
98

Thomas Jahns's avatar
Thomas Jahns committed
99
100
      msgs[i].rank = min_rank;
      msgs[i].datatype
Thomas Jahns's avatar
Thomas Jahns committed
101
102
        = xt_create_compound_datatype(num_redists, displacements, datatypes,
                                      block_lengths, comm);
103
104
105
106
      for (size_t j = 0; j < num_redists; ++j) {
        rank_pos[j]
          += (rank_pos[j] < num_ranks[j] && ranks[j][rank_pos[j]] == min_rank);
      }
107
    }
108
109
110
  }
}

Thomas Jahns's avatar
Thomas Jahns committed
111
Xt_redist
Thomas Jahns's avatar
Thomas Jahns committed
112
xt_redist_collection_static_new(Xt_redist *redists, int num_redists,
113
114
                                const MPI_Aint src_displacements[num_redists],
                                const MPI_Aint dst_displacements[num_redists],
Thomas Jahns's avatar
Thomas Jahns committed
115
116
117
118
119
120
121
122
123
124
125
126
127
                                MPI_Comm comm)
{
  return xt_redist_collection_static_custom_new(
    redists, num_redists, src_displacements, dst_displacements, comm,
    (Xt_config)&xt_default_config);
}

Xt_redist
xt_redist_collection_static_custom_new(
  Xt_redist * redists, int num_redists,
  const MPI_Aint src_displacements[num_redists],
  const MPI_Aint dst_displacements[num_redists],
  MPI_Comm comm, Xt_config config) {
128
129
  // ensure that yaxt is initialized
  assert(xt_initialized());
130

131
132
  int tag_offset;
  MPI_Comm new_comm = xt_mpi_comm_smart_dup(comm, &tag_offset);
133

134
  xt_redist_check_comms(redists, num_redists, comm);
Moritz Hanke's avatar
Moritz Hanke committed
135

136
  size_t num_redists_ = num_redists >= 0 ? (size_t)num_redists : 0;
Thomas Jahns's avatar
Thomas Jahns committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
  int *restrict ranks[2][num_redists_];
  size_t num_ranks[2][num_redists_];
  size_t nmsg[2];
  /* get lists of ranks to send/receive message to/from */
  for (size_t i = 0; i < 2; ++i)
    nmsg[i] = xt_redist_agg_msg_count(num_redists_, (enum xt_msg_direction)i,
                                      redists, num_ranks[i], ranks[i]);
  size_t nmsg_sum = nmsg[SEND] + nmsg[RECV];
  struct Xt_redist_msg *msgs = xmalloc(sizeof (*msgs) * nmsg_sum);
  for (size_t i = 0; i < 2; ++i) {
    size_t ofs = i == 0 ? 0 : nmsg[SEND];
    const MPI_Aint *disp = i == 0 ? src_displacements : dst_displacements;
    generate_msg_infos(nmsg[i], num_redists_, msgs+ofs, disp, redists,
                       num_ranks[i], (const int *restrict (*))ranks[i], new_comm,
                       (enum xt_msg_direction)i);
    free(ranks[i][0]);
  }
154

155
  Xt_redist redist_collection =
Thomas Jahns's avatar
Thomas Jahns committed
156
    xt_redist_single_array_base_custom_new(
Thomas Jahns's avatar
Thomas Jahns committed
157
      (int)nmsg[SEND], (int)nmsg[RECV], msgs, msgs+nmsg[SEND], new_comm, config);
158

Thomas Jahns's avatar
Thomas Jahns committed
159
  xt_redist_msgs_free(nmsg_sum, msgs, new_comm);
160
161
  xt_mpi_comm_smart_dedup(&new_comm, tag_offset);
  return redist_collection;
162
}
163
164
165
166
167
168
169
170
171
172

/*
 * Local Variables:
 * c-basic-offset: 2
 * coding: utf-8
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */