Line data Source code
1 : /**
2 : * \file dart_team_group.c
3 : *
4 : * Implementation of dart operations on team_group.
5 : */
6 :
7 : #include <mpi.h>
8 :
9 : #include <dash/dart/if/dart_types.h>
10 : #include <dash/dart/if/dart_team_group.h>
11 : #include <dash/dart/if/dart_initialization.h>
12 : #include <dash/dart/if/dart_locality.h>
13 :
14 : #include <dash/dart/base/logging.h>
15 : #include <dash/dart/base/macro.h>
16 : #include <dash/dart/base/assert.h>
17 : #include <dash/dart/base/locality.h>
18 :
19 : #include <dash/dart/mpi/dart_team_private.h>
20 : #include <dash/dart/mpi/dart_translation.h>
21 : #include <dash/dart/mpi/dart_group_priv.h>
22 :
23 : #include <limits.h>
24 :
25 : /* ======================================================================= *
26 : * Private Funtions *
27 : * ======================================================================= */
28 :
29 0 : dart_ret_t dart_group_init(
30 : dart_group_t *group)
31 : {
32 0 : group -> mpi_group = MPI_GROUP_EMPTY;
33 0 : return DART_OK;
34 : }
35 :
36 0 : dart_ret_t dart_group_fini(
37 : dart_group_t *group)
38 : {
39 0 : group -> mpi_group = MPI_GROUP_NULL;
40 0 : return DART_OK;
41 : }
42 :
43 0 : dart_ret_t dart_group_copy(
44 : const dart_group_t *gin,
45 : dart_group_t *gout)
46 : {
47 0 : gout -> mpi_group = gin -> mpi_group;
48 0 : return DART_OK;
49 : }
50 :
51 : #if 0
52 : dart_ret_t dart_adapt_group_union(
53 : const dart_group_t *g1,
54 : const dart_group_t *g2,
55 : dart_group_t *gout)
56 : {
57 : return MPI_Group_union(
58 : g1 -> mpi_group,
59 : g2 -> mpi_group,
60 : &(gout -> mpi_group));
61 : }
62 : #endif
63 :
64 0 : dart_ret_t dart_group_union(
65 : const dart_group_t *g1,
66 : const dart_group_t *g2,
67 : dart_group_t *gout)
68 : {
69 : /* g1 and g2 are both ordered groups. */
70 0 : if (MPI_Group_union(
71 : g1->mpi_group,
72 : g2->mpi_group,
73 : &(gout -> mpi_group)) == MPI_SUCCESS)
74 : {
75 : int i, j, k, size_in, size_out;
76 : dart_unit_t *pre_unitidsout, *post_unitidsout;;
77 :
78 : MPI_Group group_all;
79 0 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
80 0 : MPI_Group_size(gout->mpi_group, &size_out);
81 0 : if (size_out > 1) {
82 0 : MPI_Group_size(g1->mpi_group, &size_in);
83 0 : pre_unitidsout = (dart_unit_t *)malloc(
84 : size_out * sizeof (dart_unit_t));
85 0 : post_unitidsout = (dart_unit_t *)malloc(
86 : size_out * sizeof (dart_unit_t));
87 0 : dart_group_getmembers (gout, pre_unitidsout);
88 :
89 : /* Sort gout by the method of 'merge sort'. */
90 0 : i = k = 0;
91 0 : j = size_in;
92 :
93 0 : while ((i <= size_in - 1) && (j <= size_out - 1)) {
94 0 : post_unitidsout[k++] =
95 0 : (pre_unitidsout[i] <= pre_unitidsout[j])
96 0 : ? pre_unitidsout[i++]
97 0 : : pre_unitidsout[j++];
98 : }
99 0 : while (i <= size_in -1) {
100 0 : post_unitidsout[k++] = pre_unitidsout[i++];
101 : }
102 0 : while (j <= size_out -1) {
103 0 : post_unitidsout[k++] = pre_unitidsout[j++];
104 : }
105 0 : gout -> mpi_group = MPI_GROUP_EMPTY;
106 0 : MPI_Group_incl(
107 : group_all,
108 : size_out,
109 : post_unitidsout,
110 : &(gout->mpi_group));
111 0 : free (pre_unitidsout);
112 0 : free (post_unitidsout);
113 : }
114 0 : return DART_OK;
115 : }
116 0 : return DART_ERR_INVAL;
117 : }
118 :
119 0 : dart_ret_t dart_group_intersect(
120 : const dart_group_t *g1,
121 : const dart_group_t *g2,
122 : dart_group_t *gout)
123 : {
124 0 : if (MPI_Group_intersection(
125 : g1 -> mpi_group,
126 : g2 -> mpi_group,
127 : &(gout -> mpi_group)) != MPI_SUCCESS) {
128 0 : return DART_ERR_INVAL;
129 : }
130 0 : return DART_OK;
131 : }
132 :
133 0 : dart_ret_t dart_group_addmember(
134 : dart_group_t *g,
135 : dart_unit_t unitid)
136 : {
137 : int array[1];
138 : dart_group_t* group_copy, *group;
139 : MPI_Group newgroup, group_all;
140 : /* Group_all comprises all the running units. */
141 0 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
142 0 : group_copy = (dart_group_t*)malloc(sizeof(dart_group_t));
143 0 : group = (dart_group_t*)malloc(sizeof(dart_group_t));
144 0 : dart_group_copy(g, group_copy);
145 0 : array[0] = unitid;
146 0 : MPI_Group_incl(group_all, 1, array, &newgroup);
147 0 : group->mpi_group = newgroup;
148 : /* Make the new group being an ordered group. */
149 0 : dart_group_union (group_copy, group, g);
150 0 : free (group_copy);
151 0 : free (group);
152 0 : return DART_OK;
153 : }
154 :
155 0 : dart_ret_t dart_group_delmember(
156 : dart_group_t *g,
157 : dart_unit_t unitid)
158 : {
159 : int array[1];
160 : MPI_Group newgroup, group_all;
161 0 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
162 0 : array[0] = unitid;
163 0 : MPI_Group_incl(
164 : group_all,
165 : 1,
166 : array,
167 : &newgroup);
168 0 : MPI_Group_difference(
169 : g -> mpi_group,
170 : newgroup,
171 : &(g -> mpi_group));
172 0 : return DART_OK;
173 : }
174 :
175 0 : dart_ret_t dart_group_size(
176 : const dart_group_t *g,
177 : size_t *size)
178 : {
179 : int s;
180 0 : MPI_Group_size(g->mpi_group, &s);
181 0 : (*size) = s;
182 0 : return DART_OK;
183 : }
184 :
185 0 : dart_ret_t dart_group_getmembers(
186 : const dart_group_t *g,
187 : dart_unit_t *unitids)
188 : {
189 : int size, i;
190 : int *array;
191 : MPI_Group group_all;
192 0 : MPI_Group_size(g->mpi_group, &size);
193 0 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
194 0 : array = (int*) malloc(sizeof (int) * size);
195 0 : for (i = 0; i < size; i++) {
196 0 : array[i] = i;
197 : }
198 0 : MPI_Group_translate_ranks(
199 : g->mpi_group,
200 : size,
201 : array,
202 : group_all,
203 : unitids);
204 0 : free (array);
205 0 : return DART_OK;
206 : }
207 :
208 0 : dart_ret_t dart_group_split(
209 : const dart_group_t * g,
210 : size_t n,
211 : dart_group_t ** gout)
212 : {
213 : MPI_Group grouptem;
214 : int size, length, i, ranges[1][3];
215 :
216 0 : MPI_Group_size(g->mpi_group, &size);
217 :
218 : /* Ceiling division. */
219 0 : length = (size + (int)(n-1)) / ((int)(n));
220 :
221 : /* Note: split the group into chunks of subgroups. */
222 0 : for (i = 0; i < (int)n; i++) {
223 0 : if (i * length < size) {
224 0 : ranges[0][0] = i * length;
225 :
226 0 : if (i * length + length <= size) {
227 0 : ranges[0][1] = i * length + length -1;
228 : } else {
229 0 : ranges[0][1] = size - 1;
230 : }
231 0 : ranges[0][2] = 1;
232 :
233 0 : MPI_Group_range_incl(
234 : g->mpi_group,
235 : 1,
236 : ranges,
237 : &grouptem);
238 0 : (*(gout + i))->mpi_group = grouptem;
239 : } else {
240 0 : (*(gout + i))->mpi_group = MPI_GROUP_EMPTY;
241 : }
242 : }
243 0 : return DART_OK;
244 : }
245 :
246 0 : dart_ret_t dart_group_locality_split(
247 : const dart_group_t * group,
248 : dart_domain_locality_t * domain,
249 : dart_locality_scope_t scope,
250 : size_t num_groups,
251 : dart_group_t ** gout)
252 : {
253 : MPI_Group grouptem;
254 :
255 : DART_LOG_TRACE("dart_group_locality_split: split at scope %d", scope);
256 :
257 0 : dart_team_t team = domain->team;
258 :
259 : /* query domain tags of all domains in specified scope: */
260 : int num_domains;
261 : char ** domain_tags;
262 0 : DART_ASSERT_RETURNS(
263 : dart_domain_scope_tags(
264 : domain,
265 : scope,
266 : &num_domains,
267 : &domain_tags),
268 : DART_OK);
269 :
270 : DART_LOG_TRACE("dart_group_locality_split: %d domains at scope %d",
271 : num_domains, scope);
272 :
273 : /* create a group for every domain in the specified scope: */
274 :
275 0 : int total_domains_units = 0;
276 0 : dart_domain_locality_t ** domains = malloc(
277 : num_domains *
278 : sizeof(dart_domain_locality_t *));
279 0 : for (int d = 0; d < num_domains; ++d) {
280 0 : DART_ASSERT_RETURNS(
281 : dart_domain_team_locality(team, domain_tags[d], &domains[d]),
282 : DART_OK);
283 0 : total_domains_units += domains[d]->num_units;
284 :
285 : DART_LOG_TRACE("dart_group_locality_split: domains[%d]: %s",
286 : d, domain_tags[d]);
287 : DART_LOG_TRACE("dart_group_locality_split: - number of units: %d",
288 : domains[d]->num_units);
289 : }
290 : DART_LOG_TRACE("dart_group_locality_split: total number of units: %d",
291 : total_domains_units);
292 :
293 : /* TODO: splitting into more groups than domains not supported yet: */
294 0 : if (num_groups > (size_t)num_domains) {
295 0 : num_groups = num_domains;
296 : }
297 :
298 0 : if (num_groups == (size_t)num_domains) {
299 : /* one domain per group: */
300 0 : for (size_t g = 0; g < num_groups; ++g) {
301 0 : int group_num_units = domains[g]->num_units;
302 0 : int * group_team_unit_ids = domains[g]->unit_ids;
303 :
304 : /* convert relative unit ids from domain to global unit ids: */
305 0 : int * group_global_unit_ids = malloc(group_num_units * sizeof(int));
306 0 : for (int u = 0; u < group_num_units; ++u) {
307 0 : dart_team_unit_l2g(team,
308 0 : group_team_unit_ids[u],
309 0 : &group_global_unit_ids[u]);
310 : DART_LOG_TRACE("dart_group_locality_split: group[%d].units[%d] "
311 : "global unit id: %d",
312 : g, u, group_global_unit_ids[u]);
313 : }
314 :
315 0 : MPI_Group_incl(
316 : group->mpi_group,
317 : group_num_units,
318 : group_global_unit_ids,
319 : &grouptem);
320 0 : (*(gout + g))->mpi_group = grouptem;
321 :
322 0 : free(group_global_unit_ids);
323 : }
324 0 : } else if (num_groups < (size_t)num_domains) {
325 : /* Multiple domains per group. */
326 : #if 0
327 : /* TODO */
328 : /* Combine domains into groups such that the number of units in the
329 : * groups is balanced, that is: the difference of the number of units of
330 : * the groups is minimal.
331 : *
332 : * Note that this corresponds to the NP-complete partition problem
333 : * (see https://en.wikipedia.org/wiki/Partition_problem), more specific
334 : * the multi-way partition- or multiprocessor schedulig problem
335 : * (see https://en.wikipedia.org/wiki/Multiprocessor_scheduling).
336 : *
337 : * Using the "Longest Processing Time" (LPT) algorithm here.
338 : */
339 :
340 : /* number of units assigned to single groups: */
341 : int * units_in_group = calloc(num_groups * sizeof(int), sizeof(int));
342 :
343 : /* sort domains by their number of units: */
344 : typedef struct {
345 : int domain_idx;
346 : int num_units;
347 : } domain_units_t_;
348 :
349 : domain_units_t_ * sorted_domains = malloc(num_domains *
350 : sizeof(domain_units_t_));
351 :
352 : /* minimum number of units in any group: */
353 : int min_group_units = INT_MAX;
354 : /* index of group with fewest units: */
355 : int smallest_group_idx = 0;
356 : for (int d = 0; d < num_domains; ++d) {
357 : units_in_group[smallest_group_idx] = sorted_domains[d].num_units;
358 : min_group_units = INT_MAX;
359 : for (size_t g = 0; g < num_groups; ++g) {
360 : if (units_in_group[g] < min_group_units) {
361 : smallest_group_idx = g;
362 : min_group_units = units_in_group[g];
363 : }
364 : }
365 : }
366 : #else
367 : /* Preliminary implementation */
368 0 : int max_group_domains = (num_domains + (num_groups-1)) / num_groups;
369 : DART_LOG_TRACE("dart_group_locality_split: max. domains per group: %d",
370 : max_group_domains);
371 0 : for (size_t g = 0; g < num_groups; ++g) {
372 0 : int group_num_units = 0;
373 0 : int * group_team_unit_ids = NULL;
374 0 : int num_group_domains = max_group_domains;
375 0 : if ((g+1) * max_group_domains > (size_t)num_domains) {
376 0 : num_group_domains = (g * max_group_domains) - num_domains;
377 : }
378 : DART_LOG_TRACE("dart_group_locality_split: domains in group %d: %d",
379 : g, num_group_domains);
380 0 : int group_first_dom_idx = g * num_group_domains;
381 0 : int group_last_dom_idx = group_first_dom_idx + num_group_domains;
382 0 : for (int d = group_first_dom_idx; d < group_last_dom_idx; ++d) {
383 0 : group_num_units += domains[d]->num_units;
384 : }
385 0 : group_team_unit_ids = malloc(sizeof(int) * group_num_units);
386 0 : int group_unit_idx = 0;
387 0 : for (int d = group_first_dom_idx; d < group_last_dom_idx; ++d) {
388 0 : for (int u = 0; u < domains[d]->num_units; ++u) {
389 0 : group_team_unit_ids[group_unit_idx + u] = domains[d]->unit_ids[u];
390 : }
391 0 : group_unit_idx += domains[d]->num_units;
392 : }
393 :
394 : /* convert relative unit ids from domain to global unit ids: */
395 0 : int * group_global_unit_ids = malloc(group_num_units * sizeof(int));
396 0 : for (int u = 0; u < group_num_units; ++u) {
397 0 : dart_team_unit_l2g(team,
398 0 : group_team_unit_ids[u],
399 0 : &group_global_unit_ids[u]);
400 : DART_LOG_TRACE("dart_group_locality_split: group[%d].units[%d] "
401 : "global unit id: %d",
402 : g, u, group_global_unit_ids[u]);
403 : }
404 :
405 0 : MPI_Group_incl(
406 : group->mpi_group,
407 : group_num_units,
408 : group_global_unit_ids,
409 : &grouptem);
410 0 : (*(gout + g))->mpi_group = grouptem;
411 :
412 0 : free(group_team_unit_ids);
413 0 : free(group_global_unit_ids);
414 : }
415 : #endif
416 : }
417 :
418 0 : free(domains);
419 0 : free(domain_tags);
420 :
421 : DART_LOG_TRACE("dart_group_locality_split >");
422 0 : return DART_OK;
423 : }
424 :
425 0 : dart_ret_t dart_group_sizeof(size_t *size)
426 : {
427 0 : *size = sizeof(dart_group_t);
428 0 : return DART_OK;
429 : }
430 :
431 0 : dart_ret_t dart_group_ismember(
432 : const dart_group_t * g,
433 : dart_unit_t unitid,
434 : int32_t * ismember)
435 : {
436 : dart_unit_t id;
437 0 : dart_myid (&id);
438 :
439 : int i, size;
440 : dart_unit_t* ranks;
441 :
442 0 : MPI_Group_size(g->mpi_group, &size);
443 0 : ranks = (dart_unit_t *)malloc(size * sizeof(dart_unit_t));
444 0 : dart_group_getmembers (g, ranks);
445 0 : for (i = 0; i < size; i++) {
446 0 : if (ranks[i] == unitid) {
447 0 : break;
448 : }
449 : }
450 0 : *ismember = (i!=size);
451 : DART_LOG_DEBUG("dart_group_ismember : unit %2d: %s",
452 : unitid, (*ismember) ? "yes" : "no");
453 0 : return DART_OK;
454 : }
455 :
456 0 : dart_ret_t dart_team_get_group(
457 : dart_team_t teamid,
458 : dart_group_t * group)
459 : {
460 : MPI_Comm comm;
461 : uint16_t index;
462 :
463 0 : int result = dart_adapt_teamlist_convert(teamid, &index);
464 0 : if (result == -1) {
465 0 : return DART_ERR_INVAL;
466 : }
467 0 : comm = dart_teams[index];
468 0 : MPI_Comm_group(comm, &(group->mpi_group));
469 0 : return DART_OK;
470 : }
471 :
472 : /**
473 : * TODO: Differentiate units belonging to team_id and that not
474 : * belonging to team_id
475 : * within the function or outside it?
476 : * FIX: Outside it.
477 : *
478 : * The teamid stands for a superteam related to the new generated
479 : * newteam.
480 : */
481 0 : dart_ret_t dart_team_create(
482 : dart_team_t teamid,
483 : const dart_group_t * group,
484 : dart_team_t * newteam)
485 : {
486 : MPI_Comm comm;
487 : MPI_Comm subcomm;
488 : MPI_Win win;
489 : uint16_t index,
490 : unique_id;
491 : size_t size;
492 0 : dart_team_t max_teamid = -1;
493 : dart_unit_t sub_unit,
494 : unit;
495 :
496 0 : dart_myid(&unit);
497 0 : dart_size(&size);
498 0 : dart_team_myid(teamid, &sub_unit);
499 :
500 0 : int result = dart_adapt_teamlist_convert(teamid, &unique_id);
501 0 : if (result == -1) {
502 0 : return DART_ERR_INVAL;
503 : }
504 0 : comm = dart_teams[unique_id];
505 0 : subcomm = MPI_COMM_NULL;
506 :
507 0 : MPI_Comm_create(comm, group -> mpi_group, &subcomm);
508 :
509 0 : *newteam = DART_TEAM_NULL;
510 :
511 : /* Get the maximum next_availteamid among all the units belonging to
512 : * the parent team specified by 'teamid'. */
513 0 : MPI_Allreduce(
514 : &dart_next_availteamid,
515 : &max_teamid,
516 : 1,
517 : MPI_INT32_T,
518 : MPI_MAX,
519 : comm);
520 0 : dart_next_availteamid = max_teamid + 1;
521 :
522 0 : if (subcomm != MPI_COMM_NULL) {
523 0 : int result = dart_adapt_teamlist_alloc(max_teamid, &index);
524 0 : if (result == -1) {
525 0 : return DART_ERR_OTHER;
526 : }
527 : /* max_teamid is thought to be the new created team ID. */
528 0 : *newteam = max_teamid;
529 0 : dart_teams[index] = subcomm;
530 0 : MPI_Win_create_dynamic(MPI_INFO_NULL, subcomm, &win);
531 0 : dart_win_lists[index] = win;
532 : }
533 : #if 0
534 : /* Another way of generating the available teamID for the newly crated team. */
535 : if (subcomm != MPI_COMM_NULL)
536 : {
537 : /* Get the maximum next_availteamid among all the units belonging to the
538 : * created sub-communicator. */
539 : MPI_Allreduce (&next_availteamid, &max_teamid, 1, MPI_INT, MPI_MAX, subcomm);
540 : int result = dart_adapt_teamlist_alloc (max_teamid, &index);
541 :
542 : if (result == -1)
543 : {
544 : return DART_ERR_OTHER;
545 : }
546 :
547 : *newteam = max_teamid;
548 : teams[index] = subcomm;
549 : MPI_Comm_rank (subcomm, &rank);
550 :
551 : if (rank == 0)
552 : {
553 : root = sub_unit;
554 : if (sub_unit != 0)
555 : {
556 : MPI_Send (&root, 1, MPI_INT, 0, 0, comm);
557 : }
558 : }
559 :
560 : next_availteamid = max_teamid + 1;
561 : }
562 :
563 : if (sub_unit == 0)
564 : {
565 : if (root == -1)
566 : {
567 : MPI_Recv (&root, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, MPI_STATUS_IGNORE);
568 : }
569 : }
570 :
571 : MPI_Bcast (&root, 1, MPI_INT, 0, comm);
572 :
573 : /* Broadcast the calculated max_teamid to all the units not belonging to the
574 : * sub-communicator. */
575 : MPI_Bcast (&max_teamid, 1, MPI_INT, root, comm);
576 : if (subcomm == MPI_COMM_NULL)
577 : {
578 : /* 'Next_availteamid' is changed iff it is smaller than 'max_teamid + 1' */
579 : if (max_teamid + 1 > next_availteamid)
580 : {
581 : next_availteamid = max_teamid + 1;
582 : }
583 : }
584 : #endif
585 :
586 0 : if (subcomm != MPI_COMM_NULL) {
587 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
588 : int i;
589 : size_t n;
590 :
591 : MPI_Comm sharedmem_comm;
592 : MPI_Group sharedmem_group, group_all;
593 0 : MPI_Comm_split_type(
594 : subcomm,
595 : MPI_COMM_TYPE_SHARED,
596 : 1,
597 : MPI_INFO_NULL,
598 : &sharedmem_comm);
599 0 : dart_sharedmem_comm_list[index] = sharedmem_comm;
600 :
601 0 : if (sharedmem_comm != MPI_COMM_NULL) {
602 0 : MPI_Comm_size(
603 : sharedmem_comm,
604 0 : &(dart_sharedmemnode_size[index]));
605 :
606 : // dart_unit_mapping[index] = (int*)malloc (
607 : // dart_sharedmem_size[index] * sizeof (int));
608 :
609 0 : MPI_Comm_group(sharedmem_comm, &sharedmem_group);
610 0 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
611 :
612 0 : int * dart_unit_mapping = malloc(dart_sharedmemnode_size[index] *
613 : sizeof(int));
614 0 : int * sharedmem_ranks = malloc(dart_sharedmemnode_size[index] *
615 : sizeof (int));
616 0 : dart_sharedmem_table[index] = malloc(size * sizeof(int));
617 :
618 0 : for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
619 0 : sharedmem_ranks[i] = i;
620 : }
621 :
622 : // MPI_Group_translate_ranks (sharedmem_group, dart_sharedmem_size[index],
623 : // sharedmem_ranks, group_all, dart_unit_mapping[index]);
624 0 : MPI_Group_translate_ranks(
625 : sharedmem_group,
626 : dart_sharedmemnode_size[index],
627 : sharedmem_ranks,
628 : group_all,
629 : dart_unit_mapping);
630 :
631 0 : for (n = 0; n < size; n++) {
632 0 : dart_sharedmem_table[index][n] = -1;
633 : }
634 0 : for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
635 0 : dart_sharedmem_table[index][dart_unit_mapping[i]] = i;
636 : }
637 0 : free(sharedmem_ranks);
638 0 : free(dart_unit_mapping);
639 : }
640 : #endif
641 0 : MPI_Win_lock_all(0, win);
642 : DART_LOG_DEBUG("TEAMCREATE - create team %d from parent team %d",
643 : *newteam, teamid);
644 : }
645 :
646 0 : if (*newteam != DART_TEAM_NULL) {
647 0 : dart__base__locality__create(*newteam);
648 : }
649 :
650 0 : return DART_OK;
651 : }
652 :
653 0 : dart_ret_t dart_team_destroy(
654 : dart_team_t teamid)
655 : {
656 : MPI_Comm comm;
657 : MPI_Win win;
658 : uint16_t index;
659 : dart_unit_t id;
660 :
661 0 : int result = dart_adapt_teamlist_convert(teamid, &index);
662 0 : if (result == -1) {
663 0 : return DART_ERR_INVAL;
664 : }
665 0 : comm = dart_teams[index];
666 :
667 0 : dart_myid (&id);
668 :
669 : // free (dart_unit_mapping[index]);
670 :
671 : // MPI_Win_free (&(sharedmem_win_list[index]));
672 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
673 0 : free(dart_sharedmem_table[index]);
674 : #endif
675 0 : win = dart_win_lists[index];
676 0 : MPI_Win_unlock_all(win);
677 0 : MPI_Win_free(&win);
678 0 : dart_adapt_teamlist_recycle(index, result);
679 :
680 : /* -- Release the communicator associated with teamid -- */
681 0 : MPI_Comm_free (&comm);
682 :
683 : DART_LOG_DEBUG("%2d: TEAMDESTROY - destroy team %d", id, teamid);
684 0 : return DART_OK;
685 : }
686 :
687 2443 : dart_ret_t dart_myid(dart_unit_t *unitid)
688 : {
689 2443 : if (dart_initialized()) {
690 2443 : MPI_Comm_rank(MPI_COMM_WORLD, unitid);
691 : } else {
692 0 : *unitid = -1;
693 : }
694 2443 : return DART_OK;
695 : }
696 :
697 4 : dart_ret_t dart_size(size_t *size)
698 : {
699 : int s;
700 4 : MPI_Comm_size (MPI_COMM_WORLD, &s);
701 4 : (*size) = s;
702 4 : return DART_OK;
703 : }
704 :
705 1696 : dart_ret_t dart_team_myid(
706 : dart_team_t teamid,
707 : dart_unit_t * unitid)
708 : {
709 : MPI_Comm comm;
710 : uint16_t index;
711 1696 : int result = dart_adapt_teamlist_convert(teamid, &index);
712 1696 : if (result == -1)
713 : {
714 0 : return DART_ERR_INVAL;
715 : }
716 1696 : comm = dart_teams[index];
717 1696 : MPI_Comm_rank (comm, unitid);
718 :
719 1696 : return DART_OK;
720 : }
721 :
722 2156 : dart_ret_t dart_team_size(
723 : dart_team_t teamid,
724 : size_t * size)
725 : {
726 : MPI_Comm comm;
727 : uint16_t index;
728 2156 : if (teamid == DART_TEAM_NULL) {
729 0 : return DART_ERR_INVAL;
730 : }
731 2156 : int result = dart_adapt_teamlist_convert(teamid, &index);
732 2156 : if (result == -1) {
733 0 : return DART_ERR_INVAL;
734 : }
735 2156 : comm = dart_teams[index];
736 : // TODO: This should be a local operation.
737 : // Team sizes could be cached and updated in dart_team_create.
738 : int s;
739 2156 : MPI_Comm_size (comm, &s);
740 2156 : (*size) = s;
741 2156 : return DART_OK;
742 : }
743 :
744 0 : dart_ret_t dart_team_unit_l2g(
745 : dart_team_t teamid,
746 : dart_unit_t localid,
747 : dart_unit_t * globalid)
748 : {
749 : #if 0
750 : dart_unit_t *unitids;
751 : int size;
752 : int i = 0;
753 : dart_group_t group;
754 : dart_team_get_group (teamid, &group);
755 : MPI_Group_size (group.mpi_group, &size);
756 : if (localid >= size)
757 : {
758 : DART_LOG_ERROR ("Invalid localid input");
759 : return DART_ERR_INVAL;
760 : }
761 : unitids = (dart_unit_t*)malloc (sizeof(dart_unit_t) * size);
762 : dart_group_getmembers (&group, unitids);
763 :
764 : /* The unitids array is arranged in ascending order. */
765 : *globalid = unitids[localid];
766 : // printf ("globalid is %d\n", *globalid);
767 : return DART_OK;
768 : #endif
769 : int size;
770 : dart_group_t group;
771 :
772 0 : dart_team_get_group (teamid, &group);
773 0 : MPI_Group_size (group.mpi_group, &size);
774 :
775 0 : if (localid >= size) {
776 0 : DART_LOG_ERROR ("Invalid localid input: %d", localid);
777 0 : return DART_ERR_INVAL;
778 : }
779 0 : if (teamid == DART_TEAM_ALL) {
780 0 : *globalid = localid;
781 : }
782 : else {
783 : MPI_Group group_all;
784 0 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
785 0 : MPI_Group_translate_ranks(
786 : group.mpi_group,
787 : 1,
788 : &localid,
789 : group_all,
790 : globalid);
791 : }
792 :
793 0 : return DART_OK;
794 : }
795 :
796 12793890 : dart_ret_t dart_team_unit_g2l(
797 : dart_team_t teamid,
798 : dart_unit_t globalid,
799 : dart_unit_t * localid)
800 : {
801 : #if 0
802 : dart_unit_t *unitids;
803 : int size;
804 : int i;
805 : dart_group_t group;
806 : dart_team_get_group (teamid, &group);
807 : MPI_Group_size (group.mpi_group, &size);
808 : unitids = (dart_unit_t *)malloc (sizeof (dart_unit_t) * size);
809 :
810 : dart_group_getmembers (&group, unitids);
811 :
812 :
813 : for (i = 0; (i < size) && (unitids[i] < globalid); i++);
814 :
815 : if ((i == size) || (unitids[i] > globalid))
816 : {
817 : *localid = -1;
818 : return DART_OK;
819 : }
820 :
821 : *localid = i;
822 : return DART_OK;
823 : #endif
824 12793890 : if(teamid == DART_TEAM_ALL) {
825 12793890 : *localid = globalid;
826 : }
827 : else {
828 : dart_group_t group;
829 : MPI_Group group_all;
830 0 : dart_team_get_group(teamid, &group);
831 0 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
832 0 : MPI_Group_translate_ranks(
833 : group_all,
834 : 1,
835 : &globalid,
836 : group.mpi_group,
837 : localid);
838 : }
839 12793890 : return DART_OK;
840 : }
|
