LCOV - code coverage report
Current view: top level - home/fuchsto/workspaces/dash-development/dart-impl/mpi/src - dart_team_group.c (source / functions) Hit Total Coverage
Test: coverage.info.cleaned Lines: 26 259 10.0 %
Date: 2016-06-29 12:30:40 Functions: 5 22 22.7 %

          Line data    Source code
       1             : /**
       2             :  *  \file dart_team_group.c
       3             :  *
       4             :  *  Implementation of dart operations on team_group.
       5             :  */
       6             : 
       7             : #include <mpi.h>
       8             : 
       9             : #include <dash/dart/if/dart_types.h>
      10             : #include <dash/dart/if/dart_team_group.h>
      11             : #include <dash/dart/if/dart_initialization.h>
      12             : #include <dash/dart/if/dart_locality.h>
      13             : 
      14             : #include <dash/dart/base/logging.h>
      15             : #include <dash/dart/base/macro.h>
      16             : #include <dash/dart/base/assert.h>
      17             : #include <dash/dart/base/locality.h>
      18             : 
      19             : #include <dash/dart/mpi/dart_team_private.h>
      20             : #include <dash/dart/mpi/dart_translation.h>
      21             : #include <dash/dart/mpi/dart_group_priv.h>
      22             : 
      23             : #include <limits.h>
      24             : 
      25             : /* ======================================================================= *
      26             :  * Private Funtions                                                        *
      27             :  * ======================================================================= */
      28             : 
      29           0 : dart_ret_t dart_group_init(
      30             :   dart_group_t *group)
      31             : {
      32           0 :   group -> mpi_group = MPI_GROUP_EMPTY;
      33           0 :   return DART_OK;
      34             : }
      35             : 
      36           0 : dart_ret_t dart_group_fini(
      37             :   dart_group_t *group)
      38             : {
      39           0 :   group -> mpi_group = MPI_GROUP_NULL;
      40           0 :   return DART_OK;
      41             : }
      42             : 
      43           0 : dart_ret_t dart_group_copy(
      44             :   const dart_group_t *gin,
      45             :   dart_group_t *gout)
      46             : {
      47           0 :   gout -> mpi_group = gin -> mpi_group;
      48           0 :   return DART_OK;
      49             : }
      50             : 
      51             : #if 0
      52             : dart_ret_t dart_adapt_group_union(
      53             :   const dart_group_t *g1,
      54             :   const dart_group_t *g2,
      55             :   dart_group_t *gout)
      56             : {
      57             :   return MPI_Group_union(
      58             :     g1 -> mpi_group,
      59             :     g2 -> mpi_group,
      60             :     &(gout -> mpi_group));
      61             : }
      62             : #endif
      63             : 
      64           0 : dart_ret_t dart_group_union(
      65             :   const dart_group_t *g1,
      66             :   const dart_group_t *g2,
      67             :   dart_group_t *gout)
      68             : {
      69             :   /* g1 and g2 are both ordered groups. */
      70           0 :   if (MPI_Group_union(
      71             :               g1->mpi_group,
      72             :               g2->mpi_group,
      73             :               &(gout -> mpi_group)) == MPI_SUCCESS)
      74             :   {
      75             :     int i, j, k, size_in, size_out;
      76             :     dart_unit_t *pre_unitidsout, *post_unitidsout;;
      77             : 
      78             :     MPI_Group group_all;
      79           0 :     MPI_Comm_group(MPI_COMM_WORLD, &group_all);
      80           0 :     MPI_Group_size(gout->mpi_group, &size_out);
      81           0 :     if (size_out > 1) {
      82           0 :       MPI_Group_size(g1->mpi_group, &size_in);
      83           0 :       pre_unitidsout  = (dart_unit_t *)malloc(
      84             :                           size_out * sizeof (dart_unit_t));
      85           0 :       post_unitidsout = (dart_unit_t *)malloc(
      86             :                           size_out * sizeof (dart_unit_t));
      87           0 :       dart_group_getmembers (gout, pre_unitidsout);
      88             : 
      89             :       /* Sort gout by the method of 'merge sort'. */
      90           0 :       i = k = 0;
      91           0 :       j = size_in;
      92             : 
      93           0 :       while ((i <= size_in - 1) && (j <= size_out - 1)) {
      94           0 :         post_unitidsout[k++] =
      95           0 :           (pre_unitidsout[i] <= pre_unitidsout[j])
      96           0 :           ? pre_unitidsout[i++]
      97           0 :           : pre_unitidsout[j++];
      98             :       }
      99           0 :       while (i <= size_in -1) {
     100           0 :         post_unitidsout[k++] = pre_unitidsout[i++];
     101             :       }
     102           0 :       while (j <= size_out -1) {
     103           0 :         post_unitidsout[k++] = pre_unitidsout[j++];
     104             :       }
     105           0 :       gout -> mpi_group = MPI_GROUP_EMPTY;
     106           0 :       MPI_Group_incl(
     107             :         group_all,
     108             :         size_out,
     109             :         post_unitidsout,
     110             :         &(gout->mpi_group));
     111           0 :       free (pre_unitidsout);
     112           0 :       free (post_unitidsout);
     113             :     }
     114           0 :     return DART_OK;
     115             :   }
     116           0 :   return DART_ERR_INVAL;
     117             : }
     118             : 
     119           0 : dart_ret_t dart_group_intersect(
     120             :   const dart_group_t *g1,
     121             :   const dart_group_t *g2,
     122             :   dart_group_t *gout)
     123             : {
     124           0 :   if (MPI_Group_intersection(
     125             :            g1 -> mpi_group,
     126             :            g2 -> mpi_group,
     127             :            &(gout -> mpi_group)) != MPI_SUCCESS) {
     128           0 :     return DART_ERR_INVAL;
     129             :   }
     130           0 :   return DART_OK;
     131             : }
     132             : 
     133           0 : dart_ret_t dart_group_addmember(
     134             :   dart_group_t *g,
     135             :   dart_unit_t unitid)
     136             : {
     137             :   int array[1];
     138             :   dart_group_t* group_copy, *group;
     139             :   MPI_Group     newgroup, group_all;
     140             :   /* Group_all comprises all the running units. */
     141           0 :   MPI_Comm_group(MPI_COMM_WORLD, &group_all);
     142           0 :   group_copy = (dart_group_t*)malloc(sizeof(dart_group_t));
     143           0 :   group      = (dart_group_t*)malloc(sizeof(dart_group_t));
     144           0 :   dart_group_copy(g, group_copy);
     145           0 :   array[0]   = unitid;
     146           0 :   MPI_Group_incl(group_all, 1, array, &newgroup);
     147           0 :   group->mpi_group = newgroup;
     148             :   /* Make the new group being an ordered group. */
     149           0 :   dart_group_union (group_copy, group, g);
     150           0 :   free (group_copy);
     151           0 :   free (group);
     152           0 :   return DART_OK;
     153             : }
     154             : 
     155           0 : dart_ret_t dart_group_delmember(
     156             :   dart_group_t *g,
     157             :   dart_unit_t unitid)
     158             : {
     159             :   int array[1];
     160             :   MPI_Group newgroup, group_all;
     161           0 :   MPI_Comm_group(MPI_COMM_WORLD, &group_all);
     162           0 :   array[0] = unitid;
     163           0 :   MPI_Group_incl(
     164             :     group_all,
     165             :     1,
     166             :     array,
     167             :     &newgroup);
     168           0 :   MPI_Group_difference(
     169             :     g -> mpi_group,
     170             :     newgroup,
     171             :     &(g -> mpi_group));
     172           0 :   return DART_OK;
     173             : }
     174             : 
     175           0 : dart_ret_t dart_group_size(
     176             :   const dart_group_t *g,
     177             :   size_t *size)
     178             : {
     179             :   int s;
     180           0 :   MPI_Group_size(g->mpi_group, &s);
     181           0 :   (*size) = s;
     182           0 :   return DART_OK;
     183             : }
     184             : 
     185           0 : dart_ret_t dart_group_getmembers(
     186             :   const dart_group_t *g,
     187             :   dart_unit_t *unitids)
     188             : {
     189             :   int size, i;
     190             :   int *array;
     191             :   MPI_Group group_all;
     192           0 :   MPI_Group_size(g->mpi_group, &size);
     193           0 :   MPI_Comm_group(MPI_COMM_WORLD, &group_all);
     194           0 :   array = (int*) malloc(sizeof (int) * size);
     195           0 :   for (i = 0; i < size; i++) {
     196           0 :     array[i] = i;
     197             :   }
     198           0 :   MPI_Group_translate_ranks(
     199             :     g->mpi_group,
     200             :     size,
     201             :     array,
     202             :     group_all,
     203             :     unitids);
     204           0 :   free (array);
     205           0 :   return DART_OK;
     206             : }
     207             : 
     208           0 : dart_ret_t dart_group_split(
     209             :   const dart_group_t  * g,
     210             :   size_t                n,
     211             :   dart_group_t       ** gout)
     212             : {
     213             :   MPI_Group grouptem;
     214             :   int size, length, i, ranges[1][3];
     215             : 
     216           0 :   MPI_Group_size(g->mpi_group, &size);
     217             : 
     218             :   /* Ceiling division. */
     219           0 :   length = (size + (int)(n-1)) / ((int)(n));
     220             : 
     221             :   /* Note: split the group into chunks of subgroups. */
     222           0 :   for (i = 0; i < (int)n; i++) {
     223           0 :     if (i * length < size) {
     224           0 :       ranges[0][0] = i * length;
     225             : 
     226           0 :       if (i * length + length <= size) {
     227           0 :         ranges[0][1] = i * length + length -1;
     228             :       } else {
     229           0 :         ranges[0][1] = size - 1;
     230             :       }
     231           0 :       ranges[0][2] = 1;
     232             : 
     233           0 :       MPI_Group_range_incl(
     234             :         g->mpi_group,
     235             :         1,
     236             :         ranges,
     237             :         &grouptem);
     238           0 :       (*(gout + i))->mpi_group = grouptem;
     239             :     } else {
     240           0 :       (*(gout + i))->mpi_group = MPI_GROUP_EMPTY;
     241             :     }
     242             :   }
     243           0 :   return DART_OK;
     244             : }
     245             : 
     246           0 : dart_ret_t dart_group_locality_split(
     247             :   const dart_group_t      * group,
     248             :   dart_domain_locality_t  * domain,
     249             :   dart_locality_scope_t     scope,
     250             :   size_t                    num_groups,
     251             :   dart_group_t           ** gout)
     252             : {
     253             :   MPI_Group grouptem;
     254             : 
     255             :   DART_LOG_TRACE("dart_group_locality_split: split at scope %d", scope);
     256             : 
     257           0 :   dart_team_t team = domain->team;
     258             : 
     259             :   /* query domain tags of all domains in specified scope: */
     260             :   int     num_domains;
     261             :   char ** domain_tags;
     262           0 :   DART_ASSERT_RETURNS(
     263             :     dart_domain_scope_tags(
     264             :       domain,
     265             :       scope,
     266             :       &num_domains,
     267             :       &domain_tags),
     268             :     DART_OK);
     269             : 
     270             :   DART_LOG_TRACE("dart_group_locality_split: %d domains at scope %d",
     271             :                  num_domains, scope);
     272             : 
     273             :   /* create a group for every domain in the specified scope: */
     274             : 
     275           0 :   int total_domains_units           = 0;
     276           0 :   dart_domain_locality_t ** domains = malloc(
     277             :                                         num_domains *
     278             :                                         sizeof(dart_domain_locality_t *));
     279           0 :   for (int d = 0; d < num_domains; ++d) {
     280           0 :     DART_ASSERT_RETURNS(
     281             :       dart_domain_team_locality(team, domain_tags[d], &domains[d]),
     282             :       DART_OK);
     283           0 :     total_domains_units += domains[d]->num_units;
     284             : 
     285             :     DART_LOG_TRACE("dart_group_locality_split: domains[%d]: %s",
     286             :                    d, domain_tags[d]);
     287             :     DART_LOG_TRACE("dart_group_locality_split: - number of units: %d",
     288             :                    domains[d]->num_units);
     289             :   }
     290             :   DART_LOG_TRACE("dart_group_locality_split: total number of units: %d",
     291             :                  total_domains_units);
     292             : 
     293             :   /* TODO: splitting into more groups than domains not supported yet: */
     294           0 :   if (num_groups > (size_t)num_domains) {
     295           0 :     num_groups = num_domains;
     296             :   }
     297             : 
     298           0 :   if (num_groups == (size_t)num_domains) {
     299             :     /* one domain per group: */
     300           0 :     for (size_t g = 0; g < num_groups; ++g) {
     301           0 :       int   group_num_units       = domains[g]->num_units;
     302           0 :       int * group_team_unit_ids   = domains[g]->unit_ids;
     303             : 
     304             :       /* convert relative unit ids from domain to global unit ids: */
     305           0 :       int * group_global_unit_ids = malloc(group_num_units * sizeof(int));
     306           0 :       for (int u = 0; u < group_num_units; ++u) {
     307           0 :         dart_team_unit_l2g(team,
     308           0 :                            group_team_unit_ids[u],
     309           0 :                            &group_global_unit_ids[u]);
     310             :         DART_LOG_TRACE("dart_group_locality_split: group[%d].units[%d] "
     311             :                        "global unit id: %d",
     312             :                        g, u, group_global_unit_ids[u]);
     313             :       }
     314             : 
     315           0 :       MPI_Group_incl(
     316             :         group->mpi_group,
     317             :         group_num_units,
     318             :         group_global_unit_ids,
     319             :         &grouptem);
     320           0 :       (*(gout + g))->mpi_group = grouptem;
     321             : 
     322           0 :       free(group_global_unit_ids);
     323             :     }
     324           0 :   } else if (num_groups < (size_t)num_domains) {
     325             :     /* Multiple domains per group. */
     326             : #if 0
     327             :     /* TODO */
     328             :     /* Combine domains into groups such that the number of units in the
     329             :      * groups is balanced, that is: the difference of the number of units of
     330             :      * the groups is minimal.
     331             :      *
     332             :      * Note that this corresponds to the NP-complete partition problem
     333             :      * (see https://en.wikipedia.org/wiki/Partition_problem), more specific
     334             :      * the multi-way partition- or multiprocessor schedulig problem
     335             :      * (see https://en.wikipedia.org/wiki/Multiprocessor_scheduling).
     336             :      *
     337             :      * Using the "Longest Processing Time" (LPT) algorithm here.
     338             :      */
     339             : 
     340             :     /* number of units assigned to single groups: */
     341             :     int * units_in_group = calloc(num_groups * sizeof(int), sizeof(int));
     342             : 
     343             :     /* sort domains by their number of units: */
     344             :     typedef struct {
     345             :       int domain_idx;
     346             :       int num_units;
     347             :     } domain_units_t_;
     348             : 
     349             :     domain_units_t_ * sorted_domains = malloc(num_domains *
     350             :                                               sizeof(domain_units_t_));
     351             : 
     352             :     /* minimum number of units in any group: */
     353             :     int min_group_units    = INT_MAX;
     354             :     /* index of group with fewest units: */
     355             :     int smallest_group_idx = 0;
     356             :     for (int d = 0; d < num_domains; ++d) {
     357             :       units_in_group[smallest_group_idx] = sorted_domains[d].num_units;
     358             :       min_group_units = INT_MAX;
     359             :       for (size_t g = 0; g < num_groups; ++g) {
     360             :         if (units_in_group[g] < min_group_units) {
     361             :           smallest_group_idx = g;
     362             :           min_group_units    = units_in_group[g];
     363             :         }
     364             :       }
     365             :     }
     366             : #else
     367             :     /* Preliminary implementation */
     368           0 :     int max_group_domains = (num_domains + (num_groups-1)) / num_groups;
     369             :     DART_LOG_TRACE("dart_group_locality_split: max. domains per group: %d",
     370             :                    max_group_domains);
     371           0 :     for (size_t g = 0; g < num_groups; ++g) {
     372           0 :       int   group_num_units     = 0;
     373           0 :       int * group_team_unit_ids = NULL;
     374           0 :       int   num_group_domains   = max_group_domains;
     375           0 :       if ((g+1) * max_group_domains > (size_t)num_domains) {
     376           0 :         num_group_domains = (g * max_group_domains) - num_domains;
     377             :       }
     378             :       DART_LOG_TRACE("dart_group_locality_split: domains in group %d: %d",
     379             :                      g, num_group_domains);
     380           0 :       int group_first_dom_idx = g * num_group_domains;
     381           0 :       int group_last_dom_idx  = group_first_dom_idx + num_group_domains;
     382           0 :       for (int d = group_first_dom_idx; d < group_last_dom_idx; ++d) {
     383           0 :         group_num_units += domains[d]->num_units;
     384             :       }
     385           0 :       group_team_unit_ids = malloc(sizeof(int) * group_num_units);
     386           0 :       int group_unit_idx = 0;
     387           0 :       for (int d = group_first_dom_idx; d < group_last_dom_idx; ++d) {
     388           0 :         for (int u = 0; u < domains[d]->num_units; ++u) {
     389           0 :           group_team_unit_ids[group_unit_idx + u] = domains[d]->unit_ids[u];
     390             :         }
     391           0 :         group_unit_idx += domains[d]->num_units;
     392             :       }
     393             : 
     394             :       /* convert relative unit ids from domain to global unit ids: */
     395           0 :       int * group_global_unit_ids = malloc(group_num_units * sizeof(int));
     396           0 :       for (int u = 0; u < group_num_units; ++u) {
     397           0 :         dart_team_unit_l2g(team,
     398           0 :                            group_team_unit_ids[u],
     399           0 :                            &group_global_unit_ids[u]);
     400             :         DART_LOG_TRACE("dart_group_locality_split: group[%d].units[%d] "
     401             :                        "global unit id: %d",
     402             :                        g, u, group_global_unit_ids[u]);
     403             :       }
     404             : 
     405           0 :       MPI_Group_incl(
     406             :         group->mpi_group,
     407             :         group_num_units,
     408             :         group_global_unit_ids,
     409             :         &grouptem);
     410           0 :       (*(gout + g))->mpi_group = grouptem;
     411             : 
     412           0 :       free(group_team_unit_ids);
     413           0 :       free(group_global_unit_ids);
     414             :     }
     415             : #endif
     416             :   }
     417             : 
     418           0 :   free(domains);
     419           0 :   free(domain_tags);
     420             : 
     421             :   DART_LOG_TRACE("dart_group_locality_split >");
     422           0 :   return DART_OK;
     423             : }
     424             : 
     425           0 : dart_ret_t dart_group_sizeof(size_t *size)
     426             : {
     427           0 :   *size = sizeof(dart_group_t);
     428           0 :   return DART_OK;
     429             : }
     430             : 
     431           0 : dart_ret_t dart_group_ismember(
     432             :   const dart_group_t * g,
     433             :   dart_unit_t          unitid,
     434             :   int32_t            * ismember)
     435             : {
     436             :   dart_unit_t id;
     437           0 :   dart_myid (&id);
     438             : 
     439             :   int           i, size;
     440             :   dart_unit_t*  ranks;
     441             : 
     442           0 :   MPI_Group_size(g->mpi_group, &size);
     443           0 :   ranks = (dart_unit_t *)malloc(size * sizeof(dart_unit_t));
     444           0 :   dart_group_getmembers (g, ranks);
     445           0 :   for (i = 0; i < size; i++) {
     446           0 :     if (ranks[i] == unitid) {
     447           0 :       break;
     448             :     }
     449             :   }
     450           0 :   *ismember = (i!=size);
     451             :   DART_LOG_DEBUG("dart_group_ismember : unit %2d: %s",
     452             :                  unitid, (*ismember) ? "yes" : "no");
     453           0 :   return DART_OK;
     454             : }
     455             : 
     456           0 : dart_ret_t dart_team_get_group(
     457             :   dart_team_t    teamid,
     458             :   dart_group_t * group)
     459             : {
     460             :   MPI_Comm comm;
     461             :   uint16_t index;
     462             : 
     463           0 :   int result = dart_adapt_teamlist_convert(teamid, &index);
     464           0 :   if (result == -1) {
     465           0 :     return DART_ERR_INVAL;
     466             :   }
     467           0 :   comm = dart_teams[index];
     468           0 :   MPI_Comm_group(comm, &(group->mpi_group));
     469           0 :   return DART_OK;
     470             : }
     471             : 
     472             : /**
     473             :  *  TODO: Differentiate units belonging to team_id and that not
     474             :  *  belonging to team_id
     475             :  *  within the function or outside it?
     476             :  *  FIX: Outside it.
     477             :  *
     478             :  *  The teamid stands for a superteam related to the new generated
     479             :  *  newteam.
     480             :  */
     481           0 : dart_ret_t dart_team_create(
     482             :   dart_team_t          teamid,
     483             :   const dart_group_t * group,
     484             :   dart_team_t        * newteam)
     485             : {
     486             :   MPI_Comm    comm;
     487             :   MPI_Comm    subcomm;
     488             :   MPI_Win     win;
     489             :   uint16_t    index,
     490             :               unique_id;
     491             :   size_t      size;
     492           0 :   dart_team_t max_teamid = -1;
     493             :   dart_unit_t sub_unit,
     494             :               unit;
     495             : 
     496           0 :   dart_myid(&unit);
     497           0 :   dart_size(&size);
     498           0 :   dart_team_myid(teamid, &sub_unit);
     499             : 
     500           0 :   int result = dart_adapt_teamlist_convert(teamid, &unique_id);
     501           0 :   if (result == -1) {
     502           0 :     return DART_ERR_INVAL;
     503             :   }
     504           0 :   comm    = dart_teams[unique_id];
     505           0 :   subcomm = MPI_COMM_NULL;
     506             : 
     507           0 :   MPI_Comm_create(comm, group -> mpi_group, &subcomm);
     508             : 
     509           0 :   *newteam = DART_TEAM_NULL;
     510             : 
     511             :   /* Get the maximum next_availteamid among all the units belonging to
     512             :    * the parent team specified by 'teamid'. */
     513           0 :   MPI_Allreduce(
     514             :     &dart_next_availteamid,
     515             :     &max_teamid,
     516             :     1,
     517             :     MPI_INT32_T,
     518             :     MPI_MAX,
     519             :     comm);
     520           0 :   dart_next_availteamid = max_teamid + 1;
     521             : 
     522           0 :   if (subcomm != MPI_COMM_NULL) {
     523           0 :     int result = dart_adapt_teamlist_alloc(max_teamid, &index);
     524           0 :     if (result == -1) {
     525           0 :       return DART_ERR_OTHER;
     526             :     }
     527             :     /* max_teamid is thought to be the new created team ID. */
     528           0 :     *newteam = max_teamid;
     529           0 :     dart_teams[index] = subcomm;
     530           0 :     MPI_Win_create_dynamic(MPI_INFO_NULL, subcomm, &win);
     531           0 :     dart_win_lists[index] = win;
     532             :   }
     533             : #if 0
     534             :   /* Another way of generating the available teamID for the newly crated team. */
     535             :   if (subcomm != MPI_COMM_NULL)
     536             :   {
     537             :     /* Get the maximum next_availteamid among all the units belonging to the
     538             :      * created sub-communicator. */
     539             :     MPI_Allreduce (&next_availteamid, &max_teamid, 1, MPI_INT, MPI_MAX, subcomm);
     540             :     int result = dart_adapt_teamlist_alloc (max_teamid, &index);
     541             : 
     542             :     if (result == -1)
     543             :     {
     544             :       return DART_ERR_OTHER;
     545             :     }
     546             : 
     547             :     *newteam = max_teamid;
     548             :     teams[index] = subcomm;
     549             :     MPI_Comm_rank (subcomm, &rank);
     550             : 
     551             :     if (rank == 0)
     552             :     {
     553             :       root = sub_unit;
     554             :       if (sub_unit != 0)
     555             :       {
     556             :         MPI_Send (&root, 1, MPI_INT, 0, 0, comm);
     557             :       }
     558             :     }
     559             : 
     560             :     next_availteamid = max_teamid + 1;
     561             :   }
     562             : 
     563             :   if (sub_unit == 0)
     564             :   {
     565             :     if (root == -1)
     566             :     {
     567             :       MPI_Recv (&root, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, MPI_STATUS_IGNORE);
     568             :     }
     569             :   }
     570             : 
     571             :   MPI_Bcast (&root, 1, MPI_INT, 0, comm);
     572             : 
     573             :   /* Broadcast the calculated max_teamid to all the units not belonging to the
     574             :    * sub-communicator. */
     575             :   MPI_Bcast (&max_teamid, 1, MPI_INT, root, comm);
     576             :   if (subcomm == MPI_COMM_NULL)
     577             :   {
     578             :     /* 'Next_availteamid' is changed iff it is smaller than 'max_teamid + 1' */
     579             :     if (max_teamid + 1 > next_availteamid)
     580             :     {
     581             :       next_availteamid = max_teamid + 1;
     582             :     }
     583             :   }
     584             : #endif
     585             : 
     586           0 :   if (subcomm != MPI_COMM_NULL) {
     587             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
     588             :     int    i;
     589             :     size_t n;
     590             : 
     591             :     MPI_Comm sharedmem_comm;
     592             :     MPI_Group sharedmem_group, group_all;
     593           0 :     MPI_Comm_split_type(
     594             :       subcomm,
     595             :       MPI_COMM_TYPE_SHARED,
     596             :       1,
     597             :       MPI_INFO_NULL,
     598             :       &sharedmem_comm);
     599           0 :     dart_sharedmem_comm_list[index] = sharedmem_comm;
     600             : 
     601           0 :     if (sharedmem_comm != MPI_COMM_NULL) {
     602           0 :       MPI_Comm_size(
     603             :         sharedmem_comm,
     604           0 :         &(dart_sharedmemnode_size[index]));
     605             : 
     606             : //    dart_unit_mapping[index] = (int*)malloc (
     607             : //      dart_sharedmem_size[index] * sizeof (int));
     608             : 
     609           0 :       MPI_Comm_group(sharedmem_comm, &sharedmem_group);
     610           0 :       MPI_Comm_group(MPI_COMM_WORLD, &group_all);
     611             : 
     612           0 :       int * dart_unit_mapping     = malloc(dart_sharedmemnode_size[index] *
     613             :                                            sizeof(int));
     614           0 :       int * sharedmem_ranks       = malloc(dart_sharedmemnode_size[index] *
     615             :                                            sizeof (int));
     616           0 :       dart_sharedmem_table[index] = malloc(size * sizeof(int));
     617             : 
     618           0 :       for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
     619           0 :         sharedmem_ranks[i] = i;
     620             :       }
     621             : 
     622             : //    MPI_Group_translate_ranks (sharedmem_group, dart_sharedmem_size[index],
     623             : //        sharedmem_ranks, group_all, dart_unit_mapping[index]);
     624           0 :       MPI_Group_translate_ranks(
     625             :         sharedmem_group,
     626             :         dart_sharedmemnode_size[index],
     627             :         sharedmem_ranks,
     628             :         group_all,
     629             :         dart_unit_mapping);
     630             : 
     631           0 :       for (n = 0; n < size; n++) {
     632           0 :         dart_sharedmem_table[index][n] = -1;
     633             :       }
     634           0 :       for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
     635           0 :         dart_sharedmem_table[index][dart_unit_mapping[i]] = i;
     636             :       }
     637           0 :       free(sharedmem_ranks);
     638           0 :       free(dart_unit_mapping);
     639             :     }
     640             : #endif
     641           0 :     MPI_Win_lock_all(0, win);
     642             :     DART_LOG_DEBUG("TEAMCREATE - create team %d from parent team %d",
     643             :                    *newteam, teamid);
     644             :   }
     645             : 
     646           0 :   if (*newteam != DART_TEAM_NULL) {
     647           0 :     dart__base__locality__create(*newteam);
     648             :   }
     649             : 
     650           0 :   return DART_OK;
     651             : }
     652             : 
     653           0 : dart_ret_t dart_team_destroy(
     654             :   dart_team_t teamid)
     655             : {
     656             :   MPI_Comm comm;
     657             :   MPI_Win win;
     658             :   uint16_t index;
     659             :   dart_unit_t id;
     660             : 
     661           0 :   int result = dart_adapt_teamlist_convert(teamid, &index);
     662           0 :   if (result == -1) {
     663           0 :     return DART_ERR_INVAL;
     664             :   }
     665           0 :   comm = dart_teams[index];
     666             : 
     667           0 :   dart_myid (&id);
     668             : 
     669             : //  free (dart_unit_mapping[index]);
     670             : 
     671             : //  MPI_Win_free (&(sharedmem_win_list[index]));
     672             : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
     673           0 :   free(dart_sharedmem_table[index]);
     674             : #endif
     675           0 :   win = dart_win_lists[index];
     676           0 :   MPI_Win_unlock_all(win);
     677           0 :   MPI_Win_free(&win);
     678           0 :   dart_adapt_teamlist_recycle(index, result);
     679             : 
     680             :   /* -- Release the communicator associated with teamid -- */
     681           0 :   MPI_Comm_free (&comm);
     682             : 
     683             :   DART_LOG_DEBUG("%2d: TEAMDESTROY  - destroy team %d", id, teamid);
     684           0 :   return DART_OK;
     685             : }
     686             : 
     687        2443 : dart_ret_t dart_myid(dart_unit_t *unitid)
     688             : {
     689        2443 :   if (dart_initialized()) {
     690        2443 :     MPI_Comm_rank(MPI_COMM_WORLD, unitid);
     691             :   } else {
     692           0 :     *unitid = -1;
     693             :   }
     694        2443 :   return DART_OK;
     695             : }
     696             : 
     697           4 : dart_ret_t dart_size(size_t *size)
     698             : {
     699             :   int s;
     700           4 :   MPI_Comm_size (MPI_COMM_WORLD, &s);
     701           4 :   (*size) = s;
     702           4 :   return DART_OK;
     703             : }
     704             : 
     705        1696 : dart_ret_t dart_team_myid(
     706             :   dart_team_t   teamid,
     707             :   dart_unit_t * unitid)
     708             : {
     709             :   MPI_Comm comm;
     710             :   uint16_t index;
     711        1696 :   int result = dart_adapt_teamlist_convert(teamid, &index);
     712        1696 :   if (result == -1)
     713             :   {
     714           0 :     return DART_ERR_INVAL;
     715             :   }
     716        1696 :   comm = dart_teams[index];
     717        1696 :   MPI_Comm_rank (comm, unitid);
     718             : 
     719        1696 :   return DART_OK;
     720             : }
     721             : 
     722        2156 : dart_ret_t dart_team_size(
     723             :   dart_team_t   teamid,
     724             :   size_t      * size)
     725             : {
     726             :   MPI_Comm comm;
     727             :   uint16_t index;
     728        2156 :   if (teamid == DART_TEAM_NULL) {
     729           0 :     return DART_ERR_INVAL;
     730             :   }
     731        2156 :   int result = dart_adapt_teamlist_convert(teamid, &index);
     732        2156 :   if (result == -1) {
     733           0 :     return DART_ERR_INVAL;
     734             :   }
     735        2156 :   comm = dart_teams[index];
     736             :   // TODO: This should be a local operation.
     737             :   //       Team sizes could be cached and updated in dart_team_create.
     738             :   int s;
     739        2156 :   MPI_Comm_size (comm, &s);
     740        2156 :   (*size) = s;
     741        2156 :   return DART_OK;
     742             : }
     743             : 
     744           0 : dart_ret_t dart_team_unit_l2g(
     745             :   dart_team_t   teamid,
     746             :   dart_unit_t   localid,
     747             :   dart_unit_t * globalid)
     748             : {
     749             : #if 0
     750             :   dart_unit_t *unitids;
     751             :   int size;
     752             :   int i = 0;
     753             :   dart_group_t group;
     754             :   dart_team_get_group (teamid, &group);
     755             :   MPI_Group_size (group.mpi_group, &size);
     756             :   if (localid >= size)
     757             :   {
     758             :     DART_LOG_ERROR ("Invalid localid input");
     759             :     return DART_ERR_INVAL;
     760             :   }
     761             :   unitids = (dart_unit_t*)malloc (sizeof(dart_unit_t) * size);
     762             :   dart_group_getmembers (&group, unitids);
     763             : 
     764             :   /* The unitids array is arranged in ascending order. */
     765             :   *globalid = unitids[localid];
     766             : //  printf ("globalid is %d\n", *globalid);
     767             :   return DART_OK;
     768             : #endif
     769             :   int size;
     770             :   dart_group_t group;
     771             : 
     772           0 :   dart_team_get_group (teamid, &group);
     773           0 :   MPI_Group_size (group.mpi_group, &size);
     774             : 
     775           0 :   if (localid >= size) {
     776           0 :     DART_LOG_ERROR ("Invalid localid input: %d", localid);
     777           0 :     return DART_ERR_INVAL;
     778             :   }
     779           0 :   if (teamid == DART_TEAM_ALL) {
     780           0 :     *globalid = localid;
     781             :   }
     782             :   else {
     783             :     MPI_Group group_all;
     784           0 :     MPI_Comm_group(MPI_COMM_WORLD, &group_all);
     785           0 :     MPI_Group_translate_ranks(
     786             :       group.mpi_group,
     787             :       1,
     788             :       &localid,
     789             :       group_all,
     790             :       globalid);
     791             :   }
     792             : 
     793           0 :   return DART_OK;
     794             : }
     795             : 
     796    12793890 : dart_ret_t dart_team_unit_g2l(
     797             :   dart_team_t   teamid,
     798             :   dart_unit_t   globalid,
     799             :   dart_unit_t * localid)
     800             : {
     801             : #if 0
     802             :   dart_unit_t *unitids;
     803             :   int size;
     804             :   int i;
     805             :   dart_group_t group;
     806             :   dart_team_get_group (teamid, &group);
     807             :   MPI_Group_size (group.mpi_group, &size);
     808             :   unitids = (dart_unit_t *)malloc (sizeof (dart_unit_t) * size);
     809             : 
     810             :   dart_group_getmembers (&group, unitids);
     811             : 
     812             : 
     813             :   for (i = 0; (i < size) && (unitids[i] < globalid); i++);
     814             : 
     815             :   if ((i == size) || (unitids[i] > globalid))
     816             :   {
     817             :     *localid = -1;
     818             :     return DART_OK;
     819             :   }
     820             : 
     821             :   *localid = i;
     822             :   return DART_OK;
     823             : #endif
     824    12793890 :   if(teamid == DART_TEAM_ALL) {
     825    12793890 :     *localid = globalid;
     826             :   }
     827             :   else {
     828             :     dart_group_t group;
     829             :     MPI_Group group_all;
     830           0 :     dart_team_get_group(teamid, &group);
     831           0 :     MPI_Comm_group(MPI_COMM_WORLD, &group_all);
     832           0 :     MPI_Group_translate_ranks(
     833             :       group_all,
     834             :       1,
     835             :       &globalid,
     836             :       group.mpi_group,
     837             :       localid);
     838             :   }
     839    12793890 :   return DART_OK;
     840             : }

Generated by: LCOV version 1.12