adds YAC calls and now reads all grid data for an abitrary number of processes

src/elmer_dummy.c

@@ -2,26 +2,37 @@
 #include <stdio.h>
 #include <mpi.h>
 
+#include "yac.h"
+
 #include "elmer_grid.h"
 
 int main (int argc, char *argv[]) {
 
-  MPI_Init(NULL, NULL);
+  yac_cinit();
+
+  int comp_id;
+  yac_cread_config_yaml("coupling.yaml");
+  yac_cdef_comp("elmer", &comp_id);
+  MPI_Comm elmer_comm;
+  yac_cget_comp_comm(comp_id, &elmer_comm);
 
   int comm_rank, comm_size;
-  MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
-  MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
+  MPI_Comm_rank(elmer_comm, &comm_rank);
+  MPI_Comm_size(elmer_comm, &comm_size);
 
   if (argc != 3) {
-    fprintf(
-      stderr,
-      "wrong number of arguments\n"
-      "usage: %s grid_dir grid_name\n", argv[0]);
+    if (comm_rank == 0)
+      fprintf(
+        stderr,
+        "wrong number of arguments\n"
+        "usage: %s grid_dir num_parts\n"
+        "  - grid_dir: directory containing partitioned grid data\n"
+        "  - num_parts: number of partitions in grid_dir\n", argv[0]);
     MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
   }
 
   char const * grid_dir = argv[1];
-  char const * grid_name = argv[2];
+  int num_parts = atoi(argv[2]);
 
   int nbr_vertices;
   int nbr_cells;
@@ -33,15 +44,19 @@ int main (int argc, char *argv[]) {
   int * cell_to_vertex;
 
   read_grid(
-    grid_dir, comm_rank,
+    grid_dir, comm_rank, comm_size, num_parts,
     &nbr_vertices, &nbr_cells, &num_vertices_per_cell,
     &x_vertices, &y_vertices, &cell_ids, &vertex_ids, &cell_to_vertex);
 
-  int grid_id;
-
-  // yac_cdef_grid_unstruct(
-  //   grid_name, nbr_vertices, nbr_cells, num_vertices_per_cell,
-  //   x_vertices, y_vertices, cell_to_vertex, &grid_id);
+  int grid_id, corner_point_id;
+  yac_cdef_grid_unstruct(
+    "elmer_grid", nbr_vertices, nbr_cells, num_vertices_per_cell,
+    x_vertices, y_vertices, cell_to_vertex, &grid_id);
+  yac_cset_global_index(cell_ids, YAC_LOCATION_CELL, grid_id);
+  yac_cset_global_index(vertex_ids, YAC_LOCATION_CORNER, grid_id);
+  yac_cdef_points_unstruct(
+    grid_id, nbr_vertices, YAC_LOCATION_CORNER, x_vertices, y_vertices,
+    &corner_point_id);
 
   free(num_vertices_per_cell);
   free(vertex_ids);
@@ -50,7 +65,47 @@ int main (int argc, char *argv[]) {
   free(y_vertices);
   free(cell_to_vertex);
 
-  MPI_Finalize();
+  int temp_field_id;
+  int collection_size = 1;
+  char const * temp_field_timestep = "2";
+  yac_cdef_field(
+    "temp", comp_id, &corner_point_id, 1, collection_size,
+    temp_field_timestep, YAC_TIME_UNIT_SECOND, &temp_field_id);
+
+  yac_cenddef();
+
+  double * temp_field = malloc(nbr_vertices * sizeof(*temp_field));
+  for (int i = 0; i < nbr_vertices; ++i) temp_field[i] = 0.0;
+
+  int const nstep = 10;
+  for (int step = 0; step < nstep; ++step) {
+
+    if (yac_cget_role_from_field_id(temp_field_id) ==
+        YAC_EXCHANGE_TYPE_TARGET) {
+      int info, err;
+
+      if (comm_rank == 0) {
+        yac_cget_action(temp_field_id, &info);
+        fprintf(
+          stdout,
+          "%d: call get for field: \"%s\" datatime: %s action: %s\n",
+          step, yac_cget_field_name_from_field_id(temp_field_id),
+          yac_cget_field_datetime(temp_field_id),
+          ((info == YAC_ACTION_COUPLING) ||
+           (info == YAC_ACTION_GET_FOR_RESTART))?"couping":"none");
+      }
+      double *temp_field_collection_data[collection_size];
+      temp_field_collection_data[0] = temp_field;
+      yac_cget(
+        temp_field_id, collection_size, temp_field_collection_data,
+        &info, &err);
+    }
+  }
+
+  free(temp_field);
+
+  MPI_Comm_free(&elmer_comm);
+  yac_cfinalize();
 
   return EXIT_SUCCESS;
 }

src/elmer_grid.c

@@ -41,83 +41,149 @@ static void convert2rad(
 }
 
 void read_grid(
-  char const * grid_dir, int rank,
+  char const * grid_dir, int rank, int size, int num_parts,
   int * nbr_vertices, int * nbr_cells, int ** num_vertices_per_cell,
   double ** x_vertices, double ** y_vertices,
   int ** cell_ids, int ** vertex_ids, int ** cell_to_vertex) {
 
+  enum {NUM_VERT_PER_CELL = 3};
+
+  *nbr_vertices = 0;
+  *nbr_cells = 0;
+  *num_vertices_per_cell = NULL;
+  *x_vertices = NULL;
+  *y_vertices = NULL;
+  *cell_ids = NULL;
+  *vertex_ids = NULL;
+  *cell_to_vertex = NULL;
+
   if (!grid_dir) {
 
     fputs("invalid grid_dir\n", stderr);
     MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
   }
 
+  // distribute grid partitions across available ranks
+  int start_part_idx =
+    ((long long)num_parts * (long long)rank) / (long long)size;
+  int next_start_part_idx =
+    ((long long)num_parts * (long long)(rank+1)) / (long long)size;
+
+  struct glb2loc * glb2loc_vert = NULL;
+  struct glb2loc * glb2loc_cell_vert = NULL;
+
   size_t grid_dir_len = strlen(grid_dir);
   char header_filename[grid_dir_len + 64];
   char nodes_filename[grid_dir_len + 64];
   char elements_filename[grid_dir_len + 64];
 
-  sprintf(header_filename, "%s/part.%d.header", grid_dir, rank+1);
-  FILE * header_file = fopen(header_filename, "r");
-  sprintf(nodes_filename, "%s/part.%d.nodes", grid_dir, rank+1);
-  FILE * nodes_file = fopen(nodes_filename, "r");
-  sprintf(elements_filename, "%s/part.%d.elements", grid_dir, rank+1);
-  FILE * elements_file = fopen(elements_filename, "r");
-
-  if (!header_file || !nodes_file || !elements_file) {
-    fprintf(
-      stderr,"could not open grid files\n"
-      "header_file: %s %p\n"
-      "nodes_file: %s %p\n"
-      "elements_file: %s %p\n",
-      header_filename, header_file,
-      nodes_filename, nodes_file,
-      elements_filename, elements_file);
-    MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
-  }
+  // for all grid partitions of the local rank
+  for (int part_idx = start_part_idx; part_idx < next_start_part_idx;
+       ++part_idx) {
+
+    // open files of current grid partition
+    sprintf(header_filename, "%s/part.%d.header", grid_dir, part_idx+1);
+    FILE * header_file = fopen(header_filename, "r");
+    sprintf(nodes_filename, "%s/part.%d.nodes", grid_dir, part_idx+1);
+    FILE * nodes_file = fopen(nodes_filename, "r");
+    sprintf(elements_filename, "%s/part.%d.elements", grid_dir, part_idx+1);
+    FILE * elements_file = fopen(elements_filename, "r");
+
+    if (!header_file || !nodes_file || !elements_file) {
+      fprintf(
+        stderr,"could not open grid files\n"
+        "header_file: %s %p\n"
+        "nodes_file: %s %p\n"
+        "elements_file: %s %p\n",
+        header_filename, header_file,
+        nodes_filename, nodes_file,
+        elements_filename, elements_file);
+      MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
+    }
 
-  int nbr_edges;
-  fscanf(header_file, "%d%d%d", nbr_vertices, nbr_cells, &nbr_edges);
-
-  *x_vertices = malloc((size_t)*nbr_vertices * sizeof(**x_vertices));
-  *y_vertices = malloc((size_t)*nbr_vertices * sizeof(**y_vertices));
-  *vertex_ids = malloc((size_t)*nbr_vertices * sizeof(**vertex_ids));
-  struct glb2loc * glb2loc_vert =
-    malloc((size_t)*nbr_vertices * sizeof(*glb2loc_vert));
-  for (int i = 0; i < *nbr_vertices; ++i) {
-    int dummy;
-    double z_vertices;
+    // read header
+    int part_nbr_vertices, part_nbr_cells, part_nbr_edges;
     fscanf(
-      nodes_file, "%d%d%lf%lf%lf\n",
-      *vertex_ids + i, &dummy, *x_vertices + i, *y_vertices + i, &z_vertices);
-    glb2loc_vert[i].global_id = (*vertex_ids)[i];
-    glb2loc_vert[i].local_id = i;
+      header_file, "%d%d%d",
+      &part_nbr_vertices, &part_nbr_cells, &part_nbr_edges);
+
+    // allocate arrays for nodes
+    *x_vertices =
+      realloc(
+        *x_vertices,
+        (size_t)(*nbr_vertices + part_nbr_vertices) * sizeof(**x_vertices));
+    *y_vertices =
+      realloc(
+        *y_vertices,
+        (size_t)(*nbr_vertices + part_nbr_vertices) * sizeof(**y_vertices));
+    *vertex_ids =
+      realloc(
+        *vertex_ids,
+        (size_t)(*nbr_vertices + part_nbr_vertices) * sizeof(**vertex_ids));
+    glb2loc_vert =
+      realloc(
+        glb2loc_vert,
+        (size_t)(*nbr_vertices + part_nbr_vertices) * sizeof(*glb2loc_vert));
+
+    // read node data
+    for (int i = 0, j = *nbr_vertices; i < part_nbr_vertices; ++i, ++j) {
+      int dummy;
+      double z_vertices;
+      fscanf(
+        nodes_file, "%d%d%lf%lf%lf\n",
+        (*vertex_ids) + j, &dummy,
+        (*x_vertices) + j, *y_vertices + j, &z_vertices);
+      glb2loc_vert[j].global_id = (*vertex_ids)[j];
+      glb2loc_vert[j].local_id = j;
+    }
+    *nbr_vertices += part_nbr_vertices;
+
+    // allocate arrays for elements
+    *num_vertices_per_cell =
+      realloc(
+        *num_vertices_per_cell,
+        (size_t)(*nbr_cells + part_nbr_cells) * sizeof(**num_vertices_per_cell));
+    *cell_ids =
+      realloc(
+        *cell_ids,
+        (size_t)(*nbr_cells + part_nbr_cells) * sizeof(**cell_ids));
+    *cell_to_vertex =
+      realloc(
+        *cell_to_vertex,
+        (size_t)((*nbr_cells + part_nbr_cells) * NUM_VERT_PER_CELL) *
+        sizeof(**cell_to_vertex));
+    glb2loc_cell_vert =
+      realloc(
+        glb2loc_cell_vert,
+        (size_t)((*nbr_cells + part_nbr_cells) * NUM_VERT_PER_CELL) *
+        sizeof(*glb2loc_cell_vert));
+
+    // read element data
+    for (int i = 0, j = *nbr_cells; i < part_nbr_cells; ++i, ++j) {
+      (*num_vertices_per_cell)[j] = NUM_VERT_PER_CELL;
+      int dummy_a, dummy_b;
+      fscanf(
+        elements_file, "%d%d%d%d%d%d\n",
+        (*cell_ids) + j, &dummy_a, &dummy_b,
+        &(glb2loc_cell_vert[3 * j + 0].global_id),
+        &(glb2loc_cell_vert[3 * j + 1].global_id),
+        &(glb2loc_cell_vert[3 * j + 2].global_id));
+      glb2loc_cell_vert[3 * j + 0].local_id = 3 * j + 0;
+      glb2loc_cell_vert[3 * j + 1].local_id = 3 * j + 1;
+      glb2loc_cell_vert[3 * j + 2].local_id = 3 * j + 2;
+    }
+    *nbr_cells += part_nbr_cells;
+
+    // close files
+    fclose(elements_file);
+    fclose(nodes_file);
+    fclose(header_file);
   }
 
   // convert coordiantes to radian
   convert2rad(*x_vertices, *y_vertices, *nbr_vertices);
 
-  enum {NUM_VERT_PER_CELL = 3};
-  *num_vertices_per_cell =
-    malloc((size_t)*nbr_cells * sizeof(**num_vertices_per_cell));
-  *cell_ids = malloc((size_t)*nbr_cells * sizeof(**cell_ids));
-  *cell_to_vertex =
-    malloc(NUM_VERT_PER_CELL * (size_t)*nbr_cells * sizeof(**cell_to_vertex));
-  struct glb2loc * glb2loc_cell_vert =
-    malloc(NUM_VERT_PER_CELL * (size_t)*nbr_cells * sizeof(*glb2loc_cell_vert));
-  for (int i = 0; i < *nbr_cells; ++i) {
-    int dummy_a, dummy_b;
-    fscanf(
-      elements_file, "%d%d%d%d%d%d\n",
-      *cell_ids + i, &dummy_a, &dummy_b,
-      &(glb2loc_cell_vert[3 * i + 0].global_id),
-      &(glb2loc_cell_vert[3 * i + 1].global_id),
-      &(glb2loc_cell_vert[3 * i + 2].global_id));
-    glb2loc_cell_vert[3 * i + 0].local_id = 3 * i + 0;
-    glb2loc_cell_vert[3 * i + 1].local_id = 3 * i + 1;
-    glb2loc_cell_vert[3 * i + 2].local_id = 3 * i + 2;
-  }
-
+  // sort global to local lookup by global ids
   qsort(
     glb2loc_vert, (size_t)*nbr_vertices, sizeof(*glb2loc_vert),
     compare_glb2loc_glb);
@@ -126,6 +192,7 @@ void read_grid(
     sizeof(*glb2loc_cell_vert),
     compare_glb2loc_glb);
 
+  // for all cell vertices -> map global cell vertices to local ones
   for (size_t i = 0, j = 0; i < NUM_VERT_PER_CELL * *nbr_cells; ++i) {
 
     while ((j < *nbr_vertices) &&
@@ -138,12 +205,9 @@ void read_grid(
       MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
     }
 
-    (*cell_to_vertex)[i] = j;
+    (*cell_to_vertex)[glb2loc_cell_vert[i].local_id] = glb2loc_vert[j].local_id;
   }
 
   free(glb2loc_cell_vert);
   free(glb2loc_vert);
-  fclose(elements_file);
-  fclose(nodes_file);
-  fclose(header_file);
 }

src/elmer_grid.h

 void read_grid(
-  char const * grid_dir, int rank,
+  char const * grid_dir, int rank, int size, int num_parts,
   int * nbr_vertices, int * nbr_cells, int ** num_vertices_per_cell,
   double ** x_vertices, double ** y_vertices,
   int ** cell_ids, int ** vertex_ids, int ** cell_to_vertex);