Line data Source code
1 : /**
2 : * \file dart_initialization.c
3 : *
4 : * Implementations of the dart init and exit operations.
5 : */
6 : #include <stdio.h>
7 : #include <mpi.h>
8 :
9 : #include <dash/dart/if/dart_types.h>
10 : #include <dash/dart/if/dart_initialization.h>
11 : #include <dash/dart/if/dart_team_group.h>
12 :
13 : #include <dash/dart/mpi/dart_mpi_util.h>
14 : #include <dash/dart/mpi/dart_mem.h>
15 : #include <dash/dart/mpi/dart_team_private.h>
16 : #include <dash/dart/mpi/dart_translation.h>
17 : #include <dash/dart/mpi/dart_globmem_priv.h>
18 : #include <dash/dart/mpi/dart_locality_priv.h>
19 :
20 : #define DART_BUDDY_ORDER 24
21 :
22 : /* Global objects for dart memory management */
23 :
24 : /* Point to the base address of memory region for local allocation. */
25 : char* dart_mempool_localalloc;
26 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
27 : char**dart_sharedmem_local_baseptr_set;
28 : #endif
29 : /* Help to do memory management work for local allocation/free */
30 : struct dart_buddy * dart_localpool;
31 : int _init_by_dart = 0;
32 : int _dart_initialized = 0;
33 :
34 4 : dart_ret_t dart_init(
35 : int* argc,
36 : char*** argv)
37 : {
38 4 : if (_dart_initialized) {
39 0 : DART_LOG_ERROR("dart_init(): DART is already initialized");
40 0 : return DART_ERR_OTHER;
41 : }
42 : DART_LOG_DEBUG("dart_init()");
43 :
44 : int mpi_initialized;
45 4 : if (MPI_Initialized(&mpi_initialized) != MPI_SUCCESS) {
46 0 : DART_LOG_ERROR("dart_init(): MPI_Initialized failed");
47 0 : return DART_ERR_OTHER;
48 : }
49 4 : if (!mpi_initialized) {
50 4 : _init_by_dart = 1;
51 : DART_LOG_DEBUG("dart_init: MPI_Init");
52 4 : MPI_Init(argc, argv);
53 : }
54 :
55 : int rank;
56 : int size;
57 : uint16_t index;
58 : MPI_Win win;
59 :
60 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
61 : DART_LOG_DEBUG("dart_init: Shared memory enabled");
62 : MPI_Info win_info;
63 4 : MPI_Info_create(&win_info);
64 4 : MPI_Info_set(win_info, "alloc_shared_noncontig", "true");
65 : #endif
66 :
67 : /* Initialize the teamlist. */
68 4 : dart_adapt_teamlist_init();
69 :
70 4 : dart_next_availteamid = DART_TEAM_ALL;
71 4 : dart_memid = 1;
72 4 : dart_registermemid = -1;
73 :
74 4 : int result = dart_adapt_teamlist_alloc(
75 : DART_TEAM_ALL,
76 : &index);
77 4 : if (result == -1) {
78 0 : DART_LOG_ERROR("dart_adapt_teamlist_alloc failed");
79 0 : return DART_ERR_OTHER;
80 : }
81 4 : dart_teams[index] = MPI_COMM_WORLD;
82 :
83 : DART_LOG_DEBUG("dart_init: dart_adapt_teamlist_alloc completed, index:%d",
84 : index);
85 4 : dart_next_availteamid++;
86 :
87 : /* Create a global translation table for all
88 : * the collective global memory */
89 4 : dart_adapt_transtable_create();
90 :
91 4 : MPI_Comm_rank(MPI_COMM_WORLD, &rank);
92 4 : MPI_Comm_size(MPI_COMM_WORLD, &size);
93 4 : dart_localpool = dart_buddy_new(DART_BUDDY_ORDER);
94 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
95 : int i;
96 :
97 : /* Generate separated intra-node communicators and
98 : * reserve necessary resources for dart programm */
99 : MPI_Comm sharedmem_comm;
100 :
101 : /* Splits the communicator into subcommunicators,
102 : * each of which can create a shared memory region */
103 4 : if (MPI_Comm_split_type(
104 : MPI_COMM_WORLD,
105 : MPI_COMM_TYPE_SHARED,
106 : 1,
107 : MPI_INFO_NULL,
108 : &sharedmem_comm) != MPI_SUCCESS)
109 : {
110 0 : DART_LOG_ERROR("dart_init: MPI_Comm_split_type failed");
111 0 : return DART_ERR_OTHER;
112 : }
113 :
114 4 : dart_sharedmem_comm_list[index] = sharedmem_comm;
115 :
116 : MPI_Group group_all, sharedmem_group;
117 : char *baseptr;
118 :
119 4 : if (sharedmem_comm != MPI_COMM_NULL) {
120 : DART_LOG_DEBUG("dart_init: MPI_Win_allocate_shared(nbytes:%d)",
121 : DART_MAX_LENGTH);
122 : /* Reserve a free shared memory block for non-collective
123 : * global memory allocation. */
124 4 : int ret = MPI_Win_allocate_shared(
125 : DART_MAX_LENGTH,
126 : sizeof(char),
127 : win_info,
128 : sharedmem_comm,
129 : &(dart_mempool_localalloc),
130 : &dart_sharedmem_win_local_alloc);
131 4 : if (ret != MPI_SUCCESS) {
132 0 : DART_LOG_ERROR("dart_init: "
133 : "MPI_Win_allocate_shared failed, error %d (%s)",
134 : ret, DART__MPI__ERROR_STR(ret));
135 0 : return DART_ERR_OTHER;
136 : }
137 : DART_LOG_DEBUG("dart_init: MPI_Win_allocate_shared completed");
138 :
139 : int sharedmem_unitid;
140 4 : MPI_Comm_size(
141 : sharedmem_comm,
142 4 : &(dart_sharedmemnode_size[index]));
143 4 : MPI_Comm_rank(
144 : sharedmem_comm,
145 : &sharedmem_unitid);
146 4 : dart_sharedmem_local_baseptr_set =
147 4 : (char**)malloc(sizeof(char*) * dart_sharedmemnode_size[index]);
148 : MPI_Aint winseg_size;
149 :
150 : int disp_unit;
151 20 : for(i = 0; i < dart_sharedmemnode_size[index]; i++) {
152 16 : if (sharedmem_unitid != i) {
153 12 : MPI_Win_shared_query(
154 : dart_sharedmem_win_local_alloc,
155 : i,
156 : &winseg_size,
157 : &disp_unit,
158 : &baseptr);
159 12 : dart_sharedmem_local_baseptr_set[i] = baseptr;
160 : }
161 : else {
162 4 : dart_sharedmem_local_baseptr_set[i] = dart_mempool_localalloc;
163 : }
164 : }
165 :
166 4 : MPI_Comm_group(sharedmem_comm, &sharedmem_group);
167 4 : MPI_Comm_group(MPI_COMM_WORLD, &group_all);
168 :
169 : /* The length of this table is set to be the size
170 : * of DART_TEAM_ALL. */
171 4 : dart_sharedmem_table[index] = (int *)malloc (sizeof (int) * size);
172 :
173 4 : int* dart_unit_mapping = (int*) malloc(
174 4 : sizeof(int) * dart_sharedmemnode_size[index]);
175 4 : int* sharedmem_ranks = (int*) malloc(
176 4 : sizeof(int) * dart_sharedmemnode_size[index]);
177 :
178 20 : for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
179 16 : sharedmem_ranks[i] = i;
180 : }
181 20 : for (i = 0; i < size; i++) {
182 16 : dart_sharedmem_table[index][i] = -1;
183 : }
184 :
185 : /* Generate the set (dart_unit_mapping) of units with absolute IDs,
186 : * which are located in the same node
187 : */
188 4 : if (MPI_Group_translate_ranks(
189 : sharedmem_group,
190 : dart_sharedmemnode_size[index],
191 : sharedmem_ranks,
192 : group_all,
193 : dart_unit_mapping) != MPI_SUCCESS)
194 : {
195 0 : DART_LOG_ERROR("dart_init: MPI_Group_translate_ranks failed");
196 0 : return DART_ERR_OTHER;
197 : }
198 :
199 : /* The non-negative elements in the array
200 : * 'dart_sharedmem_table[index]' consist of a serial of units,
201 : * which are in the same node and they can communicate via
202 : * shared memory and i is the relative position in the node
203 : * for unit dart_unit_mapping[i].
204 : */
205 20 : for (i = 0; i < dart_sharedmemnode_size[index]; i++) {
206 16 : dart_sharedmem_table[index][dart_unit_mapping[i]] = i;
207 : }
208 4 : free(sharedmem_ranks);
209 4 : free(dart_unit_mapping);
210 : }
211 : #else
212 : MPI_Alloc_mem(
213 : DART_MAX_LENGTH,
214 : MPI_INFO_NULL,
215 : &dart_mempool_localalloc);
216 : #endif
217 : /* Create a single global win object for dart local
218 : * allocation based on the aboved allocated shared memory.
219 : *
220 : * Return in dart_win_local_alloc. */
221 4 : MPI_Win_create(
222 : dart_mempool_localalloc,
223 : DART_MAX_LENGTH,
224 : sizeof(char),
225 : MPI_INFO_NULL,
226 : MPI_COMM_WORLD,
227 : &dart_win_local_alloc);
228 :
229 : /* Create a dynamic win object for all the dart collective
230 : * allocation based on MPI_COMM_WORLD. Return in win. */
231 4 : MPI_Win_create_dynamic(
232 : MPI_INFO_NULL, MPI_COMM_WORLD, &win);
233 4 : dart_win_lists[index] = win;
234 :
235 : /* Start an access epoch on dart_win_local_alloc, and later
236 : * on all the units can access the memory region allocated
237 : * by the local allocation function through
238 : * dart_win_local_alloc. */
239 4 : MPI_Win_lock_all(0, dart_win_local_alloc);
240 :
241 : /* Start an access epoch on win, and later on all the units
242 : * can access the attached memory region allocated by the
243 : * collective allocation function through win. */
244 4 : MPI_Win_lock_all(0, win);
245 :
246 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
247 4 : MPI_Info_free(&win_info);
248 : #endif
249 : DART_LOG_DEBUG("dart_init: communication backend initialization finished");
250 :
251 4 : _dart_initialized = 1;
252 :
253 4 : dart__mpi__locality_init();
254 :
255 : DART_LOG_DEBUG("dart_init > initialization finished");
256 4 : return DART_OK;
257 : }
258 :
259 4 : dart_ret_t dart_exit()
260 : {
261 4 : if (!_dart_initialized) {
262 0 : DART_LOG_ERROR("dart_exit(): DART has not been initialized");
263 0 : return DART_ERR_OTHER;
264 : }
265 : uint16_t index;
266 : dart_unit_t unitid;
267 4 : dart_myid(&unitid);
268 :
269 4 : dart__mpi__locality_finalize();
270 :
271 4 : _dart_initialized = 0;
272 :
273 : DART_LOG_DEBUG("%2d: dart_exit()", unitid);
274 4 : if (dart_adapt_teamlist_convert(DART_TEAM_ALL, &index) == -1) {
275 0 : DART_LOG_ERROR("%2d: dart_exit: dart_adapt_teamlist_convert failed", unitid);
276 0 : return DART_ERR_OTHER;
277 : }
278 :
279 4 : if (MPI_Win_unlock_all(dart_win_lists[index]) != MPI_SUCCESS) {
280 0 : DART_LOG_ERROR("%2d: dart_exit: MPI_Win_unlock_all failed", unitid);
281 0 : return DART_ERR_OTHER;
282 : }
283 : /* End the shared access epoch in dart_win_local_alloc. */
284 4 : if (MPI_Win_unlock_all(dart_win_local_alloc) != MPI_SUCCESS) {
285 0 : DART_LOG_ERROR("%2d: dart_exit: MPI_Win_unlock_all failed", unitid);
286 0 : return DART_ERR_OTHER;
287 : }
288 : /* -- Free up all the resources for dart programme -- */
289 4 : MPI_Win_free(&dart_win_local_alloc);
290 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
291 4 : MPI_Win_free(&dart_sharedmem_win_local_alloc);
292 : #endif
293 4 : MPI_Win_free(&dart_win_lists[index]);
294 :
295 4 : dart_adapt_transtable_destroy();
296 4 : dart_buddy_delete(dart_localpool);
297 : #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
298 4 : free(dart_sharedmem_table[index]);
299 4 : free(dart_sharedmem_local_baseptr_set);
300 : #endif
301 4 : dart_adapt_teamlist_destroy();
302 :
303 4 : if (_init_by_dart) {
304 : DART_LOG_DEBUG("%2d: dart_exit: MPI_Finalize", unitid);
305 4 : MPI_Finalize();
306 : }
307 :
308 : DART_LOG_DEBUG("%2d: dart_exit: finalization finished", unitid);
309 :
310 4 : return DART_OK;
311 : }
312 :
313 2443 : char dart_initialized()
314 : {
315 2443 : return _dart_initialized;
316 : }
|
