# Graph Communication

# examples/09-graph_communication.jl
using Test
using MPI

MPI.Init()

comm = MPI.COMM_WORLD
size = MPI.Comm_size(comm)
rank = MPI.Comm_rank(comm)

#
# Setup the following communication graph
#
#   +-----+
#   |     |
#   v     v
#   0<-+  3
#   ^  |  ^
#   |  |  |
#   v  |  v
#   1  +--2
#   ^     |
#   |     |
#   +-----+
#
#

if rank == 0
dest   = Cint[1,3]
degree = Cint[length(dest)]
elseif rank == 1
dest   = Cint
degree = Cint[length(dest)]
elseif rank == 2
dest   = Cint[3,0,1]
degree = Cint[length(dest)]
elseif rank == 3
dest   = Cint[0,2,1]
degree = Cint[length(dest)]
end

source = Cint[rank]
graph_comm = MPI.Dist_graph_create(comm, source, degree, dest)

#
# Now send the rank across the edges.
#
# Version 1: use allgather primitive
#

send = [rank]
if rank == 0
recv = [-1, -1, -1]
elseif rank == 1
recv = [-1, -1, -1]
elseif rank == 2
recv = [-1]
elseif rank == 3
recv = [-1, -1]
end

MPI.Neighbor_allgather!(send, recv, graph_comm);

println("rank = $(rank):$(recv)")

#
# Version 2: use alltoall primitive
#

if rank == 0
send   = [rank, rank]
recv   = [-1, -1, -1]
elseif rank == 1
send = [rank]
recv   = [-1, -1, -1]
elseif rank == 2
send = [rank, rank, rank]
recv   = [-1]
elseif rank == 3
send = [rank, rank, rank]
recv   = [-1, -1]
end

MPI.Neighbor_alltoall!(UBuffer(send,1), UBuffer(recv,1), graph_comm);

println("rank = $(rank):$(recv)")

#
# Now send the rank exactly rank times across the edges.
#
if rank == 0
send       = [rank, rank,
rank, rank, rank, rank]
send_count = [2, 4]

recv   = [-1, -1, -1]
recv_count = [1, 1, 1]
elseif rank == 1
send       = [rank]
send_count = 

recv       = [-1, -1, -1, -1, -1, -1]
recv_count = [2, 2, 2]
elseif rank == 2
send       = [rank, rank, rank, rank,
rank,
rank,rank]
send_count = [4, 1, 2]

recv       = [-1, -1, -1]
recv_count = 
elseif rank == 3
send       = [rank,
rank, rank,rank,
rank, rank]
send_count = [1, 3, 2]

recv       = [-1, -1, -1, -1, -1, -1, -1, -1]
recv_count = [4, 4]
end

MPI.Neighbor_alltoallv!(VBuffer(send,send_count), VBuffer(recv,recv_count), graph_comm);
println("rank = $(rank):$(recv)")

MPI.Finalize()
> mpiexecjl -n 4 julia examples/09-graph_communication.jl
rank = 1: [3, 2, 0]rank = 3: [2, 0]

rank = 2: rank = 0: [3, 2, 1]

rank = 2: rank = 0: [3, 2, 1]rank = 3: [2, 0]

rank = 1: [3, 2, 0]

rank = 0: [3, 2, 1]rank = 3: [2, 2, 2, 2, 0, 0, 0, 0]

rank = 2: [3, 3, 3]
rank = 1: [3, 3, 2, 2, 0, 0]