communication.F90

Go to the documentation of this file.

module communication
!
!       Contains the variable definition of the processor number,
!       myID and the number of processors, nProc, which belong to the
!       group defined by the communicator ADflow_comm_world. The range
!       of processor numbers is <0..Nproc-1>, i.e. the numbering
!       starts at 0. This is done for compatibility with MPI.
!       Furthermore this module contains the communication pattern for
!       all the multigrid levels.
!
    use constants, only: inttype, realtype
    implicit none
    save
!
!       The definition of the derived data type commListType, which
!       stores the i,j and k indices as well as the block id of the
!       data to be communicated. Send lists may contain interpolants
!       since the indices may refer to a stencil, while the receive
!       list does not. All interpolations should be done on the send
!       side to keep message sizes to a minimum.
!
#ifndef USE_TAPENADE
    type sendcommlisttype
 
        ! block(..):     Local block id to which the cell/node belongs.
        !                The dimension is equal to the number of entities
        !                to be communicated with the particular processor.
        ! indices(..,3): I, j and k indices of the data to be communicated.
        !                For the first dimension, see block.
        ! interp(..,..): Interpolants for indices that represent a cell
        !                stencil (allocated only when needed, e.g. send
        !                list for overset communication).
        !                For the first dimension, see block.
 
        integer(kind=intType), pointer, dimension(:) :: block
        integer(kind=intType), pointer, dimension(:, :) :: indices
        real(kind=realtype), pointer, dimension(:, :) :: interp, interpd
        real(kind=realtype), pointer, dimension(:, :) :: xcen
    end type sendcommlisttype
 
    type recvcommlisttype
 
        ! block(..):     Local block id to which the cell/node belongs.
        !                The dimension is equal to the number of entities
        !                to be communicated with the particular processor.
        ! indices(..,3): I, j and k indices of the data to be communicated.
        !                For the first dimension, see block.
 
        integer(kind=intType), pointer, dimension(:) :: block
        integer(kind=intType), pointer, dimension(:, :) :: indices
 
    end type recvcommlisttype
!
!       The definition of the derived data type periodicDataType,
!       which stores the rotation matrix, the rotation center and the
!       translation vector of the periodic transformation, as well as
!       the halos to which this transformation must be applied.
!
    type periodicdatatype
 
        ! rotMatrix(3,3): Rotation matrix.
        ! rotCenter(3):   Coordinates of center of rotation.
        ! translation(3):  Translation vector.
 
        real(kind=realtype), dimension(3, 3) :: rotmatrix
        real(kind=realtype), dimension(3) :: rotcenter, translation
 
        ! nHalos:            # of halos to which this periodic
        !                    transformation must be applied.
        ! block(nHalos):     Local block id to which the halos belong.
        ! indices(nHalos,3): I, j and k indices of the halos in
        !                    this block.
 
        integer(kind=intType) :: nhalos
 
        integer(kind=intType), pointer, dimension(:) :: block
        integer(kind=intType), pointer, dimension(:, :) :: indices
 
    end type periodicdatatype
!
!       The definition of the derived data type commType, which
!       stores the communication pattern for a certain halo type for a
!       certain grid level.
!
    type commtype
 
        ! nProcSend:           # of procs, to whom this proc will send
        ! nProcRecv:           # of procs, from whom this proc will
        !                      receive.
        ! sendProc(nProcSend): Send processor numbers.
        ! recvProc(nProcRecv): Receive processor numbers.
 
        integer(kind=intType) :: nprocsend, nprocrecv
        integer(kind=intType), pointer, dimension(:) :: sendproc, recvproc
 
        ! nsend(nProcSend): # of entities to send to other processors.
        ! nrecv(nProcRecv): # of entities to receive from other processors.
 
        integer(kind=intType), pointer, dimension(:) :: nsend, nrecv
 
        ! nsendCum(0:NprocSend): cumulative version of nsend.
        ! nrecvCum(0:NprocRecv): cumulative version of nrecv.
 
        integer(kind=intType), pointer, dimension(:) :: nsendcum
        integer(kind=intType), pointer, dimension(:) :: nrecvcum
 
        ! indexSendProc(0:Nproc-1): index of the processors in sendProc.
        !                           If nothing is to be sent to a
        !                           processor this value is 0.
        ! indexRecvProc(0:Nproc-1): index of the processors in recvProc.
        !                           If nothing is to be received from a
        !                           processor this value is 0.
 
        integer(kind=intType), pointer, dimension(:) :: indexsendproc
        integer(kind=intType), pointer, dimension(:) :: indexrecvproc
 
        ! sendList(nProcSend): Indices and block ids to send to these
        !                      processors.
        ! recvList(nProcRecv): Indices and block ids to receive from
        !                      these processors.
 
        type(sendcommlisttype), pointer, dimension(:) :: sendlist
        type(recvcommlisttype), pointer, dimension(:) :: recvlist
 
        ! nPeriodic:               # of periodic data arrays.
        ! periodicData(nPeriodic): Periodic data and entities to which
        !                          the transformation must be applied.
 
        integer(kind=intType) :: nperiodic
        type(periodicdatatype), pointer, dimension(:) :: periodicdata
 
    end type commtype
!
!       The definition of the derived data type internalCommType,
!       which stores the memory to memory copy on this processor for a
!       certain halo type.
!
    type internalcommtype
 
        ! ncopy: # of entities to copy internally.
 
        ! donorBlock(ncopy):     Local block id of the donor cell.
        ! donorIndices(ncopy,3): The indices of the donor cell.
        ! donorInterp(ncopy,3):  Interpolants of the donor stencil.
        !                        Only allocated when needed, e.g.
        !                        overset communication).
 
        ! haloBlock(ncopy):     Local block id of the halo cell.
        ! haloIndices(ncopy,3): The indices of the halo cell.
 
        integer(kind=intType) :: ncopy
 
        integer(kind=intType), pointer, dimension(:) :: donorblock
        integer(kind=intType), pointer, dimension(:, :) :: donorindices
        real(kind=realtype), pointer, dimension(:, :) :: donorinterp, donorinterpd
        real(kind=realtype), pointer, dimension(:, :) :: xcen
 
        integer(kind=intType), pointer, dimension(:) :: haloblock
        integer(kind=intType), pointer, dimension(:, :) :: haloindices
 
        ! nPeriodic:               # of periodic data arrays.
        ! periodicData(nPeriodic): Periodic data and entities to which
        !                          the transformation must be applied.
 
        integer(kind=intType) :: nperiodic
        type(periodicdatatype), pointer, dimension(:) :: periodicdata
 
    end type internalcommtype
!
!       Variables stored in this module.
!
    ! ADflow_comm_world: The communicator of this processor group.
    ! myID:            My processor number in ADflow_comm_world.
    ! nProc:           The number of processors in ADflow_comm_world.
    integer :: adflow_comm_world, adflow_comm_self
 
    integer :: myid, nproc
 
    ! commPatternCell_1st(nLevel): The communication pattern for 1st
    !                              level cell halo's on the multiple
    !                              grids.
    ! commPatternCell_2nd(nLevel): The communication pattern for 2nd
    !                              level cell halo's (including the
    !                              1st level) on the multiple grids.
    ! commPatternNode_1st(nLevel): The communication pattern for 1st
    !                              level node halo's on the multiple
    !                              grids.
 
    type(commtype), allocatable, dimension(:) :: commpatterncell_1st
    type(commtype), allocatable, dimension(:) :: commpatterncell_2nd
    type(commtype), allocatable, dimension(:) :: commpatternnode_1st
    type(commtype), allocatable, target, dimension(:, :) :: commpatternoverset
 
    ! internalCell_1st(nLevel): Memory to memory copies for 1st level
    !                           cell halo's on the multiple grids.
    ! internalCell_2nd(nLevel): Memory to memory copies for 2nd level
    !                           cell halo's on the multiple grids.
    ! internalNode_1st(nLevel): Memory to memory copies for 1st level
    !                           node halo's on the multiple grids.
 
    type(internalcommtype), allocatable, dimension(:) :: internalcell_1st
    type(internalcommtype), allocatable, dimension(:) :: internalcell_2nd
    type(internalcommtype), allocatable, dimension(:) :: internalnode_1st
    type(internalcommtype), allocatable, target, dimension(:, :) :: internaloverset
 
    ! sendBufferSize_1to1: Size of the send buffer needed to perform
    !                      all 1 to 1 communication.
    ! recvBufferSize_1to1: Idem for the receive buffer.
    ! sendBufferSizeOver:  Size of the send buffer needed to perform
    !                      all overset communication.
    ! recvBufferSizeOver:  Idem for the receive buffer.
    ! sendBufferSize:      Size of the send buffer to perform all
    !                      possible communication.
    ! recvBufferSize:      Idem for the receive buffer.
 
    integer(kind=intType) :: sendbuffersize_1to1, sendbuffersize
    integer(kind=intType) :: recvbuffersize_1to1, recvbuffersize
    integer(kind=intType) :: sendbuffersizeover, recvbuffersizeover
 
    ! sendBuffer:   Buffer used to store the info to be send during
    !               a nonblocking communication.
    ! recvBuffer:   Buffer used to store the info to be received
    !               during a nonblocking communication.
    ! sendRequests: Array of requests for the nonblocking sends.
    ! recvRequests: Array of requests for the nonblocking receives.
 
    real(kind=realtype), allocatable, dimension(:) :: sendbuffer
    real(kind=realtype), allocatable, dimension(:) :: recvbuffer
 
    integer, allocatable, dimension(:) :: sendrequests, recvrequests
#endif
end module communication