LCOV - code coverage report
Current view: top level - home/fuchsto/workspaces/dash-development/dart-impl/mpi/src - dart_initialization.c (source / functions) Hit Total Coverage
Test: coverage.info.cleaned Lines: 81 101 80.2 %
Date: 2016-06-29 12:30:40 Functions: 3 3 100.0 %

          Line data    Source code
       1             : /**
       2             :  * \file dart_initialization.c
       3             :  *
       4             :  *  Implementations of the dart init and exit operations.
       5             :  */
       6             : #include <stdio.h>
       7             : #include <mpi.h>
       8             : 
       9             : #include <dash/dart/if/dart_types.h>
      10             : #include <dash/dart/if/dart_initialization.h>
      11             : #include <dash/dart/if/dart_team_group.h>
      12             : 
      13             : #include <dash/dart/mpi/dart_mpi_util.h>
      14             : #include <dash/dart/mpi/dart_mem.h>
      15             : #include <dash/dart/mpi/dart_team_private.h>
      16             : #include <dash/dart/mpi/dart_translation.h>
      17             : #include <dash/dart/mpi/dart_globmem_priv.h>
      18             : #include <dash/dart/mpi/dart_locality_priv.h>
      19             : 
      20             : #define DART_BUDDY_ORDER 24
      21             : 
      22             : /* Global objects for dart memory management */
      23             : 
      24             : /* Point to the base address of memory region for local allocation. */
      25             : char* dart_mempool_localalloc;
      26             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
      27             : char**dart_sharedmem_local_baseptr_set;
      28             : #endif
      29             : /* Help to do memory management work for local allocation/free */
      30             : struct dart_buddy   * dart_localpool;
      31             : int                   _init_by_dart     = 0;
      32             : int                   _dart_initialized = 0;
      33             : 
      34           4 : dart_ret_t dart_init(
      35             :   int*    argc,
      36             :   char*** argv)
      37             : {
      38           4 :   if (_dart_initialized) {
      39           0 :     DART_LOG_ERROR("dart_init(): DART is already initialized");
      40           0 :     return DART_ERR_OTHER;
      41             :   }
      42             :   DART_LOG_DEBUG("dart_init()");
      43             : 
      44             :         int mpi_initialized;
      45           4 :         if (MPI_Initialized(&mpi_initialized) != MPI_SUCCESS) {
      46           0 :     DART_LOG_ERROR("dart_init(): MPI_Initialized failed");
      47           0 :     return DART_ERR_OTHER;
      48             :   }
      49           4 :         if (!mpi_initialized) {
      50           4 :                 _init_by_dart = 1;
      51             :     DART_LOG_DEBUG("dart_init: MPI_Init");
      52           4 :                 MPI_Init(argc, argv);
      53             :         }
      54             : 
      55             :         int      rank;
      56             :   int      size;
      57             :         uint16_t index;
      58             :         MPI_Win  win;
      59             : 
      60             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
      61             :   DART_LOG_DEBUG("dart_init: Shared memory enabled");
      62             :         MPI_Info win_info;
      63           4 :         MPI_Info_create(&win_info);
      64           4 :         MPI_Info_set(win_info, "alloc_shared_noncontig", "true");
      65             : #endif
      66             : 
      67             :         /* Initialize the teamlist. */
      68           4 :         dart_adapt_teamlist_init();
      69             : 
      70           4 :         dart_next_availteamid = DART_TEAM_ALL;
      71           4 :         dart_memid = 1;
      72           4 :         dart_registermemid = -1;
      73             : 
      74           4 :         int result = dart_adapt_teamlist_alloc(
      75             :                  DART_TEAM_ALL,
      76             :                  &index);
      77           4 :         if (result == -1) {
      78           0 :     DART_LOG_ERROR("dart_adapt_teamlist_alloc failed");
      79           0 :                 return DART_ERR_OTHER;
      80             :         }
      81           4 :         dart_teams[index] = MPI_COMM_WORLD;
      82             : 
      83             :   DART_LOG_DEBUG("dart_init: dart_adapt_teamlist_alloc completed, index:%d",
      84             :                  index);
      85           4 :         dart_next_availteamid++;
      86             : 
      87             :         /* Create a global translation table for all
      88             :    * the collective global memory */
      89           4 :         dart_adapt_transtable_create();
      90             : 
      91           4 :         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
      92           4 :         MPI_Comm_size(MPI_COMM_WORLD, &size);
      93           4 :         dart_localpool = dart_buddy_new(DART_BUDDY_ORDER);
      94             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
      95             :         int i;
      96             : 
      97             :         /* Generate separated intra-node communicators and
      98             :    * reserve necessary resources for dart programm */
      99             :         MPI_Comm sharedmem_comm;
     100             : 
     101             :         /* Splits the communicator into subcommunicators,
     102             :    * each of which can create a shared memory region */
     103           4 :         if (MPI_Comm_split_type(
     104             :         MPI_COMM_WORLD,
     105             :         MPI_COMM_TYPE_SHARED,
     106             :         1,
     107             :         MPI_INFO_NULL,
     108             :         &sharedmem_comm) != MPI_SUCCESS)
     109             :   {
     110           0 :     DART_LOG_ERROR("dart_init: MPI_Comm_split_type failed");
     111           0 :     return DART_ERR_OTHER;
     112             :   }
     113             : 
     114           4 :         dart_sharedmem_comm_list[index] = sharedmem_comm;
     115             : 
     116             :         MPI_Group group_all, sharedmem_group;
     117             :         char *baseptr;
     118             : 
     119           4 :         if (sharedmem_comm != MPI_COMM_NULL) {
     120             :     DART_LOG_DEBUG("dart_init: MPI_Win_allocate_shared(nbytes:%d)",
     121             :                    DART_MAX_LENGTH);
     122             :                 /* Reserve a free shared memory block for non-collective
     123             :      * global memory allocation. */
     124           4 :                 int ret = MPI_Win_allocate_shared(
     125             :                 DART_MAX_LENGTH,
     126             :                 sizeof(char),
     127             :                 win_info,
     128             :                 sharedmem_comm,
     129             :                 &(dart_mempool_localalloc),
     130             :                 &dart_sharedmem_win_local_alloc);
     131           4 :     if (ret != MPI_SUCCESS) {
     132           0 :       DART_LOG_ERROR("dart_init: "
     133             :                      "MPI_Win_allocate_shared failed, error %d (%s)",
     134             :                      ret, DART__MPI__ERROR_STR(ret));
     135           0 :       return DART_ERR_OTHER;
     136             :     }
     137             :     DART_LOG_DEBUG("dart_init: MPI_Win_allocate_shared completed");
     138             : 
     139             :                 int sharedmem_unitid;
     140           4 :                 MPI_Comm_size(
     141             :       sharedmem_comm,
     142           4 :       &(dart_sharedmemnode_size[index]));
     143           4 :                 MPI_Comm_rank(
     144             :       sharedmem_comm,
     145             :       &sharedmem_unitid);
     146           4 :                 dart_sharedmem_local_baseptr_set =
     147           4 :       (char**)malloc(sizeof(char*) * dart_sharedmemnode_size[index]);
     148             :                 MPI_Aint winseg_size;
     149             : 
     150             :                 int disp_unit;
     151          20 :                 for(i = 0; i < dart_sharedmemnode_size[index]; i++) {
     152          16 :                         if (sharedmem_unitid != i) {
     153          12 :                                 MPI_Win_shared_query(
     154             :           dart_sharedmem_win_local_alloc,
     155             :           i,
     156             :           &winseg_size,
     157             :           &disp_unit,
     158             :           &baseptr);
     159          12 :                                 dart_sharedmem_local_baseptr_set[i] = baseptr;
     160             :       }
     161             :                         else {
     162           4 :         dart_sharedmem_local_baseptr_set[i] = dart_mempool_localalloc;
     163             :       }
     164             :                 }
     165             : 
     166           4 :                 MPI_Comm_group(sharedmem_comm, &sharedmem_group);
     167           4 :                 MPI_Comm_group(MPI_COMM_WORLD, &group_all);
     168             : 
     169             :                 /* The length of this table is set to be the size
     170             :      * of DART_TEAM_ALL. */
     171           4 :                 dart_sharedmem_table[index] = (int *)malloc (sizeof (int) * size);
     172             : 
     173           4 :                 int* dart_unit_mapping = (int*) malloc(
     174           4 :                                sizeof(int) * dart_sharedmemnode_size[index]);
     175           4 :                 int* sharedmem_ranks   = (int*) malloc(
     176           4 :                                sizeof(int) * dart_sharedmemnode_size[index]);
     177             : 
     178          20 :                 for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
     179          16 :                         sharedmem_ranks[i] = i;
     180             :                 }
     181          20 :                 for (i = 0; i < size; i++) {
     182          16 :                         dart_sharedmem_table[index][i] = -1;
     183             :                 }
     184             : 
     185             :                 /* Generate the set (dart_unit_mapping) of units with absolute IDs,
     186             :                  * which are located in the same node
     187             :                  */
     188           4 :                 if (MPI_Group_translate_ranks(
     189             :           sharedmem_group,
     190             :           dart_sharedmemnode_size[index],
     191             :           sharedmem_ranks,
     192             :           group_all,
     193             :           dart_unit_mapping) != MPI_SUCCESS)
     194             :     {
     195           0 :       DART_LOG_ERROR("dart_init: MPI_Group_translate_ranks failed");
     196           0 :       return DART_ERR_OTHER;
     197             :     }
     198             : 
     199             :                 /* The non-negative elements in the array
     200             :      * 'dart_sharedmem_table[index]' consist of a serial of units,
     201             :      * which are in the same node and they can communicate via
     202             :      * shared memory and i is the relative position in the node
     203             :      * for unit dart_unit_mapping[i].
     204             :                  */
     205          20 :                 for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
     206          16 :                         dart_sharedmem_table[index][dart_unit_mapping[i]] = i;
     207             :                 }
     208           4 :                 free(sharedmem_ranks);
     209           4 :                 free(dart_unit_mapping);
     210             :         }
     211             : #else
     212             :         MPI_Alloc_mem(
     213             :     DART_MAX_LENGTH,
     214             :     MPI_INFO_NULL,
     215             :     &dart_mempool_localalloc);
     216             : #endif
     217             :         /* Create a single global win object for dart local
     218             :    * allocation based on the aboved allocated shared memory.
     219             :          *
     220             :          * Return in dart_win_local_alloc. */
     221           4 :         MPI_Win_create(
     222             :     dart_mempool_localalloc,
     223             :     DART_MAX_LENGTH,
     224             :     sizeof(char),
     225             :     MPI_INFO_NULL,
     226             :                 MPI_COMM_WORLD,
     227             :     &dart_win_local_alloc);
     228             : 
     229             :         /* Create a dynamic win object for all the dart collective
     230             :    * allocation based on MPI_COMM_WORLD. Return in win. */
     231           4 :         MPI_Win_create_dynamic(
     232             :     MPI_INFO_NULL, MPI_COMM_WORLD, &win);
     233           4 :         dart_win_lists[index] = win;
     234             : 
     235             :         /* Start an access epoch on dart_win_local_alloc, and later
     236             :    * on all the units can access the memory region allocated
     237             :    * by the local allocation function through
     238             :    * dart_win_local_alloc. */
     239           4 :         MPI_Win_lock_all(0, dart_win_local_alloc);
     240             : 
     241             :         /* Start an access epoch on win, and later on all the units
     242             :    * can access the attached memory region allocated by the
     243             :    * collective allocation function through win. */
     244           4 :         MPI_Win_lock_all(0, win);
     245             : 
     246             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
     247           4 :         MPI_Info_free(&win_info);
     248             : #endif
     249             :         DART_LOG_DEBUG("dart_init: communication backend initialization finished");
     250             : 
     251           4 :   _dart_initialized = 1;
     252             : 
     253           4 :   dart__mpi__locality_init();
     254             : 
     255             :         DART_LOG_DEBUG("dart_init > initialization finished");
     256           4 :         return DART_OK;
     257             : }
     258             : 
     259           4 : dart_ret_t dart_exit()
     260             : {
     261           4 :   if (!_dart_initialized) {
     262           0 :     DART_LOG_ERROR("dart_exit(): DART has not been initialized");
     263           0 :     return DART_ERR_OTHER;
     264             :   }
     265             :         uint16_t    index;
     266             :         dart_unit_t unitid;
     267           4 :         dart_myid(&unitid);
     268             : 
     269           4 :   dart__mpi__locality_finalize();
     270             : 
     271           4 :   _dart_initialized = 0;
     272             : 
     273             :         DART_LOG_DEBUG("%2d: dart_exit()", unitid);
     274           4 :         if (dart_adapt_teamlist_convert(DART_TEAM_ALL, &index) == -1) {
     275           0 :     DART_LOG_ERROR("%2d: dart_exit: dart_adapt_teamlist_convert failed", unitid);
     276           0 :     return DART_ERR_OTHER;
     277             :   }
     278             : 
     279           4 :         if (MPI_Win_unlock_all(dart_win_lists[index]) != MPI_SUCCESS) {
     280           0 :     DART_LOG_ERROR("%2d: dart_exit: MPI_Win_unlock_all failed", unitid);
     281           0 :     return DART_ERR_OTHER;
     282             :   }
     283             :         /* End the shared access epoch in dart_win_local_alloc. */
     284           4 :         if (MPI_Win_unlock_all(dart_win_local_alloc) != MPI_SUCCESS) {
     285           0 :     DART_LOG_ERROR("%2d: dart_exit: MPI_Win_unlock_all failed", unitid);
     286           0 :     return DART_ERR_OTHER;
     287             :   }
     288             :         /* -- Free up all the resources for dart programme -- */
     289           4 :         MPI_Win_free(&dart_win_local_alloc);
     290             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
     291           4 :         MPI_Win_free(&dart_sharedmem_win_local_alloc);
     292             : #endif
     293           4 :         MPI_Win_free(&dart_win_lists[index]);
     294             : 
     295           4 :         dart_adapt_transtable_destroy();
     296           4 :         dart_buddy_delete(dart_localpool);
     297             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
     298           4 :         free(dart_sharedmem_table[index]);
     299           4 :         free(dart_sharedmem_local_baseptr_set);
     300             : #endif
     301           4 :         dart_adapt_teamlist_destroy();
     302             : 
     303           4 :         if (_init_by_dart) {
     304             :     DART_LOG_DEBUG("%2d: dart_exit: MPI_Finalize", unitid);
     305           4 :                 MPI_Finalize();
     306             :   }
     307             : 
     308             :         DART_LOG_DEBUG("%2d: dart_exit: finalization finished", unitid);
     309             : 
     310           4 :         return DART_OK;
     311             : }
     312             : 
     313        2443 : char dart_initialized()
     314             : {
     315        2443 :   return _dart_initialized;
     316             : }

Generated by: LCOV version 1.12