combine arrays on different cores

The way to combine arrays in different cores to a big array by MPI is using MPI_Allgatherv. An example I would like to test is like this.

Core 1 : 1

Core 2 : 2 2

Core 3 : 3 3 3

Core 4 : 4 4 4 4

Core 5 ...

After combine all the arrays, I will get

1 2 2 3 3 3 4 4 4 4 5 5 ...

Here is the code(test_allgatherv.c) and Makefile for this calculation. Pay attention to displ_list.

#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>

int main(int argc, char *argv[])
{

  int my_rank; // MPI rank
  int size; // MPI size
  int i; 
  int *local_array; // array in local core 
  int local_num; // num of elements in local array
  int *local_num_list; // the list of local_num 
  int *displ_list; // the displ_list of global_array
  int *global_array; // array in global
  int global_num; // sum of local_num

  // MPI related variables
  MPI_Init (&argc, &argv);
  MPI_Comm_rank (MPI_COMM_WORLD, &my_rank);
  MPI_Comm_size(MPI_COMM_WORLD, &size);

  // generate local array
  local_num = my_rank + 1;
  local_array = (int *) malloc (local_num*sizeof(int));
  for (i=0;i<local_num;i++) {
    local_array[i] = local_num;
    printf("%d \n", local_num);
  }

  // calculate global_num = sum[local_num]
  MPI_Allreduce(&local_num, &global_num, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);

  // using MPI_Allgatherv the calculation of displ_list is a little tricky
  global_array = (int *) malloc (global_num*sizeof(int));
  displ_list = (int *) malloc (size*sizeof(int));
  local_num_list = (int *) malloc (size*sizeof(int));
  MPI_Allgather(&local_num, 1, MPI_INT, local_num_list, 1, MPI_INT, MPI_COMM_WORLD);
  displ_list[0] = 0;
  for (i=1;i<size;i++) {
    displ_list[i] = displ_list[i-1] + local_num_list[i-1];
  }
  MPI_Allgatherv(local_array, local_num, MPI_INT, global_array, local_num_list, displ_list, MPI_INT, MPI_COMM_WORLD);

  // output the global_array
  if (my_rank == 0){
    printf("after MPI_Allgatherv\n");
    for (i=0;i<global_num;i++){
      printf("%d %d\n",i , global_array[i]);
    }
  }
  MPI_Finalize();
  return 0;
}

And the Makefile.

CC=mpicc
CFLAGS=-c -Wall
LDFLAGS=
SOURCES=test_allgatherv.c
OBJECTS=$(SOURCES:.c=.o)
EXECUTABLE=test_allgatherv

all: $(SOURCES) $(EXECUTABLE)

$(EXECUTABLE): $(OBJECTS)
    $(CC) $(LDFLAGS) $(OBJECTS) -o $@

.c.o:
    $(CC) $(CFLAGS) $< -o $@

After compiling, run the job by

mpirun -n 8 ./test_allgatherv

I got the correct result

1 
2 
2 
4 
4 
4 
4 
5 
5 
5 
5 
5 
8 
8 
8 
8 
8 
8 
8 
8 
3 
3 
3 
6 
6 
6 
6 
6 
6 
7 
7 
7 
7 
7 
7 
7 
after MPI_Allgatherv
0 1
1 2
2 2
3 3
4 3
5 3
6 4
7 4
8 4
9 4
10 5
11 5
12 5
13 5
14 5
15 6
16 6
17 6
18 6
19 6
20 6
21 7
22 7
23 7
24 7
25 7
26 7
27 7
28 8
29 8
30 8
31 8
32 8
33 8
34 8
35 8