MPI 进阶通信：派生数据类型、通信子与单边通信

#include <mpi.h>
#include <stdlib.h>

/* 从 8x8 矩阵中取出第 0 列（步长 8，取 8 个元素） */
int rows = 8, cols = 8;
double *A = malloc(sizeof(double) * rows * cols);

MPI_Datatype coltype;
MPI_Type_vector(rows,        /* count: 8 个块 */
                1,           /* blocklength: 每块 1 个元素 */
                cols,        /* stride: 块间步长 8 个元素 */
                MPI_DOUBLE,
                &coltype);
MPI_Type_commit(&coltype);

MPI_Send(A, 1, coltype, 1, 0, MPI_COMM_WORLD);
MPI_Type_free(&coltype);

int sizes[2]    = {N, N};       /* 全局数组维度 */
int subsizes[2] = {n, n};       /* 子块维度 */
int starts[2]   = {i0, j0};     /* 子块起点 */
MPI_Datatype sub;
MPI_Type_create_subarray(2, sizes, subsizes, starts,
                         MPI_ORDER_C, MPI_DOUBLE, &sub);
MPI_Type_commit(&sub);

MPI_Aint lb, extent;
MPI_Type_get_extent(structtype, &lb, &extent);
/* 若 extent != sizeof(struct), 用 create_resized 重设 */
MPI_Type_create_resized(oldtype, 0, sizeof(MyStruct), &newtype);
MPI_Type_commit(&newtype);

int position = 0;
MPI_Pack(src, 10, MPI_DOUBLE, buffer, bufsize, &position, MPI_COMM_WORLD);
MPI_Pack(other, 5, MPI_INT, buffer, bufsize, &position, MPI_COMM_WORLD);
MPI_Send(buffer, position, MPI_PACKED, dest, 0, MPI_COMM_WORLD);

int color = rank % 2;              /* 奇偶分成两组 */
int key   = rank;
MPI_Comm newcomm;
MPI_Comm_split(MPI_COMM_WORLD, color, key, &newcomm);

int newrank, newsize;
MPI_Comm_rank(newcomm, &newrank);
MPI_Comm_size(newcomm, &newsize);
/* color 传 MPI_UNDEFINED 的进程不进任何新子域 */

MPI_Comm nodecomm;
MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED,
                    0, MPI_INFO_NULL, &nodecomm);

MPI_Group world_group, io_group, compute_group;
MPI_Comm_group(MPI_COMM_WORLD, &world_group);

int io_ranks[4] = {0, 1, 2, 3};
MPI_Group_incl(world_group, 4, io_ranks, &io_group);
MPI_Group_difference(world_group, io_group, &compute_group);

MPI_Comm io_comm, compute_comm;
MPI_Comm_create(MPI_COMM_WORLD, io_group, &io_comm);
MPI_Comm_create(MPI_COMM_WORLD, compute_group, &compute_comm);

MPI_Comm nodecomm;
MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED,
                    0, MPI_INFO_NULL, &nodecomm);

int noderank, nodesize;
MPI_Comm_rank(nodecomm, &noderank);
MPI_Comm_size(nodecomm, &nodesize);

/* 每个进程在共享段中申请一块可被同节点其他进程直接访问的内存 */
MPI_Win shmwin;
double *shm;
MPI_Win_allocate_shared(shm_local_bytes, sizeof(double),
                        MPI_INFO_NULL, nodecomm, &shm, &shmwin);

/* 查询邻居 rank 的共享段基址，直接指针访问，无需 MPI 调用 */
double *peer_base;
MPI_Aint peer_size;
MPI_Win_shared_query(shmwin, peer_rank, &peer_size, &peer_base, &peer_base);

double *base;
MPI_Win win;
MPI_Win_allocate(bufsize * sizeof(double), sizeof(double),
                 MPI_INFO_NULL, MPI_COMM_WORLD, &base, &win);
/* 每个进程暴露自己的 base，其他进程可远程访问 */

/* 模式一：主动目标（Passive Target），目标进程不参与 */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target_rank, 0, win);
MPI_Put(local, n, MPI_DOUBLE, target_rank, offset, n, MPI_DOUBLE, win);
MPI_Get(remote_result, n, MPI_DOUBLE, target_rank, offset, n, MPI_DOUBLE, win);
MPI_Win_unlock(target_rank, win);

/* 模式二：集体同步，所有进程进入同一 epoch */
MPI_Win_fence(0, win);
MPI_Put(local, n, MPI_DOUBLE, target, 0, n, MPI_DOUBLE, win);
MPI_Win_fence(0, win);   /* 此处保证 Put 已完成并可见 */

MPI_Put(buf, n, MPI_DOUBLE, tgt, 0, n, MPI_DOUBLE, win);
MPI_Win_flush(tgt, win);        /* 该目标的 Put 已排空 */
/* 若远端要读本地暴露内存，还需 MPI_Win_sync + MPI_Barrier */

MPI_Request req[4];
MPI_Send_init(send_n, n, MPI_DOUBLE, north, 0, comm, &req[0]);
MPI_Recv_init(recv_n, n, MPI_DOUBLE, south, 0, comm, &req[1]);
MPI_Send_init(send_s, n, MPI_DOUBLE, south, 1, comm, &req[2]);
MPI_Recv_init(recv_s, n, MPI_DOUBLE, north, 1, comm, &req[3]);

for (int step = 0; step < nsteps; step++) {
    MPI_Startall(4, req);
    compute_interior(domain);
    MPI_Waitall(4, req, MPI_STATUSES_IGNORE);
    compute_boundary(domain);
}

/* 每个邻居发送/接收的数据量可不同（用数组描述） */
MPI_Neighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes,
                       recvbuf, recvcounts, rdispls, recvtypes,
                       cart_comm);

MPI_Comm workercomm;
int errcodes[NWORKERS];
MPI_Comm_spawn("./worker", MPI_ARGV_NULL, NWORKERS,
               MPI_INFO_NULL, 0, MPI_COMM_SELF,
               &workercomm, errcodes);

/* 父进程与子进程通过 workercomm 通信（其中 rank 0 为父端） */
MPI_Send(task, n, MPI_INT, 0, 0, workercomm);

MPI_Comm parent;
MPI_Comm_get_parent(&parent);
/* 此后 parent 上的 rank 0 即父进程 */