LLVM OpenMP* Runtime Library
kmp_tasking.cpp
1 /*
2  * kmp_tasking.cpp -- OpenMP 3.0 tasking support.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_i18n.h"
15 #include "kmp_itt.h"
16 #include "kmp_stats.h"
17 #include "kmp_wait_release.h"
18 #include "kmp_taskdeps.h"
19 
20 #if OMPT_SUPPORT
21 #include "ompt-specific.h"
22 #endif
23 
24 /* forward declaration */
25 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
26  kmp_info_t *this_thr);
27 static void __kmp_alloc_task_deque(kmp_info_t *thread,
28  kmp_thread_data_t *thread_data);
29 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
30  kmp_task_team_t *task_team);
31 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
32 
33 #ifdef BUILD_TIED_TASK_STACK
34 
35 // __kmp_trace_task_stack: print the tied tasks from the task stack in order
36 // from top do bottom
37 //
38 // gtid: global thread identifier for thread containing stack
39 // thread_data: thread data for task team thread containing stack
40 // threshold: value above which the trace statement triggers
41 // location: string identifying call site of this function (for trace)
42 static void __kmp_trace_task_stack(kmp_int32 gtid,
43  kmp_thread_data_t *thread_data,
44  int threshold, char *location) {
45  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
46  kmp_taskdata_t **stack_top = task_stack->ts_top;
47  kmp_int32 entries = task_stack->ts_entries;
48  kmp_taskdata_t *tied_task;
49 
50  KA_TRACE(
51  threshold,
52  ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
53  "first_block = %p, stack_top = %p \n",
54  location, gtid, entries, task_stack->ts_first_block, stack_top));
55 
56  KMP_DEBUG_ASSERT(stack_top != NULL);
57  KMP_DEBUG_ASSERT(entries > 0);
58 
59  while (entries != 0) {
60  KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0]);
61  // fix up ts_top if we need to pop from previous block
62  if (entries & TASK_STACK_INDEX_MASK == 0) {
63  kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top);
64 
65  stack_block = stack_block->sb_prev;
66  stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
67  }
68 
69  // finish bookkeeping
70  stack_top--;
71  entries--;
72 
73  tied_task = *stack_top;
74 
75  KMP_DEBUG_ASSERT(tied_task != NULL);
76  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
77 
78  KA_TRACE(threshold,
79  ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
80  "stack_top=%p, tied_task=%p\n",
81  location, gtid, entries, stack_top, tied_task));
82  }
83  KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0]);
84 
85  KA_TRACE(threshold,
86  ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n",
87  location, gtid));
88 }
89 
90 // __kmp_init_task_stack: initialize the task stack for the first time
91 // after a thread_data structure is created.
92 // It should not be necessary to do this again (assuming the stack works).
93 //
94 // gtid: global thread identifier of calling thread
95 // thread_data: thread data for task team thread containing stack
96 static void __kmp_init_task_stack(kmp_int32 gtid,
97  kmp_thread_data_t *thread_data) {
98  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
99  kmp_stack_block_t *first_block;
100 
101  // set up the first block of the stack
102  first_block = &task_stack->ts_first_block;
103  task_stack->ts_top = (kmp_taskdata_t **)first_block;
104  memset((void *)first_block, '\0',
105  TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *));
106 
107  // initialize the stack to be empty
108  task_stack->ts_entries = TASK_STACK_EMPTY;
109  first_block->sb_next = NULL;
110  first_block->sb_prev = NULL;
111 }
112 
113 // __kmp_free_task_stack: free the task stack when thread_data is destroyed.
114 //
115 // gtid: global thread identifier for calling thread
116 // thread_data: thread info for thread containing stack
117 static void __kmp_free_task_stack(kmp_int32 gtid,
118  kmp_thread_data_t *thread_data) {
119  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
120  kmp_stack_block_t *stack_block = &task_stack->ts_first_block;
121 
122  KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY);
123  // free from the second block of the stack
124  while (stack_block != NULL) {
125  kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL;
126 
127  stack_block->sb_next = NULL;
128  stack_block->sb_prev = NULL;
129  if (stack_block != &task_stack->ts_first_block) {
130  __kmp_thread_free(thread,
131  stack_block); // free the block, if not the first
132  }
133  stack_block = next_block;
134  }
135  // initialize the stack to be empty
136  task_stack->ts_entries = 0;
137  task_stack->ts_top = NULL;
138 }
139 
140 // __kmp_push_task_stack: Push the tied task onto the task stack.
141 // Grow the stack if necessary by allocating another block.
142 //
143 // gtid: global thread identifier for calling thread
144 // thread: thread info for thread containing stack
145 // tied_task: the task to push on the stack
146 static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread,
147  kmp_taskdata_t *tied_task) {
148  // GEH - need to consider what to do if tt_threads_data not allocated yet
149  kmp_thread_data_t *thread_data =
150  &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)];
151  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
152 
153  if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) {
154  return; // Don't push anything on stack if team or team tasks are serialized
155  }
156 
157  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
158  KMP_DEBUG_ASSERT(task_stack->ts_top != NULL);
159 
160  KA_TRACE(20,
161  ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n",
162  gtid, thread, tied_task));
163  // Store entry
164  *(task_stack->ts_top) = tied_task;
165 
166  // Do bookkeeping for next push
167  task_stack->ts_top++;
168  task_stack->ts_entries++;
169 
170  if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
171  // Find beginning of this task block
172  kmp_stack_block_t *stack_block =
173  (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE);
174 
175  // Check if we already have a block
176  if (stack_block->sb_next !=
177  NULL) { // reset ts_top to beginning of next block
178  task_stack->ts_top = &stack_block->sb_next->sb_block[0];
179  } else { // Alloc new block and link it up
180  kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc(
181  thread, sizeof(kmp_stack_block_t));
182 
183  task_stack->ts_top = &new_block->sb_block[0];
184  stack_block->sb_next = new_block;
185  new_block->sb_prev = stack_block;
186  new_block->sb_next = NULL;
187 
188  KA_TRACE(
189  30,
190  ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n",
191  gtid, tied_task, new_block));
192  }
193  }
194  KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid,
195  tied_task));
196 }
197 
198 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return
199 // the task, just check to make sure it matches the ending task passed in.
200 //
201 // gtid: global thread identifier for the calling thread
202 // thread: thread info structure containing stack
203 // tied_task: the task popped off the stack
204 // ending_task: the task that is ending (should match popped task)
205 static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread,
206  kmp_taskdata_t *ending_task) {
207  // GEH - need to consider what to do if tt_threads_data not allocated yet
208  kmp_thread_data_t *thread_data =
209  &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)];
210  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
211  kmp_taskdata_t *tied_task;
212 
213  if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) {
214  // Don't pop anything from stack if team or team tasks are serialized
215  return;
216  }
217 
218  KMP_DEBUG_ASSERT(task_stack->ts_top != NULL);
219  KMP_DEBUG_ASSERT(task_stack->ts_entries > 0);
220 
221  KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid,
222  thread));
223 
224  // fix up ts_top if we need to pop from previous block
225  if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
226  kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top);
227 
228  stack_block = stack_block->sb_prev;
229  task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
230  }
231 
232  // finish bookkeeping
233  task_stack->ts_top--;
234  task_stack->ts_entries--;
235 
236  tied_task = *(task_stack->ts_top);
237 
238  KMP_DEBUG_ASSERT(tied_task != NULL);
239  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
240  KMP_DEBUG_ASSERT(tied_task == ending_task); // If we built the stack correctly
241 
242  KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid,
243  tied_task));
244  return;
245 }
246 #endif /* BUILD_TIED_TASK_STACK */
247 
248 // returns 1 if new task is allowed to execute, 0 otherwise
249 // checks Task Scheduling constraint (if requested) and
250 // mutexinoutset dependencies if any
251 static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained,
252  const kmp_taskdata_t *tasknew,
253  const kmp_taskdata_t *taskcurr) {
254  if (is_constrained && (tasknew->td_flags.tiedness == TASK_TIED)) {
255  // Check if the candidate obeys the Task Scheduling Constraints (TSC)
256  // only descendant of all deferred tied tasks can be scheduled, checking
257  // the last one is enough, as it in turn is the descendant of all others
258  kmp_taskdata_t *current = taskcurr->td_last_tied;
259  KMP_DEBUG_ASSERT(current != NULL);
260  // check if the task is not suspended on barrier
261  if (current->td_flags.tasktype == TASK_EXPLICIT ||
262  current->td_taskwait_thread > 0) { // <= 0 on barrier
263  kmp_int32 level = current->td_level;
264  kmp_taskdata_t *parent = tasknew->td_parent;
265  while (parent != current && parent->td_level > level) {
266  // check generation up to the level of the current task
267  parent = parent->td_parent;
268  KMP_DEBUG_ASSERT(parent != NULL);
269  }
270  if (parent != current)
271  return false;
272  }
273  }
274  // Check mutexinoutset dependencies, acquire locks
275  kmp_depnode_t *node = tasknew->td_depnode;
276  if (UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
277  for (int i = 0; i < node->dn.mtx_num_locks; ++i) {
278  KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
279  if (__kmp_test_lock(node->dn.mtx_locks[i], gtid))
280  continue;
281  // could not get the lock, release previous locks
282  for (int j = i - 1; j >= 0; --j)
283  __kmp_release_lock(node->dn.mtx_locks[j], gtid);
284  return false;
285  }
286  // negative num_locks means all locks acquired successfully
287  node->dn.mtx_num_locks = -node->dn.mtx_num_locks;
288  }
289  return true;
290 }
291 
292 // __kmp_realloc_task_deque:
293 // Re-allocates a task deque for a particular thread, copies the content from
294 // the old deque and adjusts the necessary data structures relating to the
295 // deque. This operation must be done with the deque_lock being held
296 static void __kmp_realloc_task_deque(kmp_info_t *thread,
297  kmp_thread_data_t *thread_data) {
298  kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td);
299  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == size);
300  kmp_int32 new_size = 2 * size;
301 
302  KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "
303  "%d] for thread_data %p\n",
304  __kmp_gtid_from_thread(thread), size, new_size, thread_data));
305 
306  kmp_taskdata_t **new_deque =
307  (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *));
308 
309  int i, j;
310  for (i = thread_data->td.td_deque_head, j = 0; j < size;
311  i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++)
312  new_deque[j] = thread_data->td.td_deque[i];
313 
314  __kmp_free(thread_data->td.td_deque);
315 
316  thread_data->td.td_deque_head = 0;
317  thread_data->td.td_deque_tail = size;
318  thread_data->td.td_deque = new_deque;
319  thread_data->td.td_deque_size = new_size;
320 }
321 
322 // __kmp_push_task: Add a task to the thread's deque
323 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
324  kmp_info_t *thread = __kmp_threads[gtid];
325  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
326 
327  // We don't need to map to shadow gtid if it is already hidden helper thread
328  if (taskdata->td_flags.hidden_helper && !KMP_HIDDEN_HELPER_THREAD(gtid)) {
329  gtid = KMP_GTID_TO_SHADOW_GTID(gtid);
330  thread = __kmp_threads[gtid];
331  }
332 
333  kmp_task_team_t *task_team = thread->th.th_task_team;
334  kmp_int32 tid = __kmp_tid_from_gtid(gtid);
335  kmp_thread_data_t *thread_data;
336 
337  KA_TRACE(20,
338  ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata));
339 
340  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
341  // untied task needs to increment counter so that the task structure is not
342  // freed prematurely
343  kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count);
344  KMP_DEBUG_USE_VAR(counter);
345  KA_TRACE(
346  20,
347  ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n",
348  gtid, counter, taskdata));
349  }
350 
351  // The first check avoids building task_team thread data if serialized
352  if (UNLIKELY(taskdata->td_flags.task_serial)) {
353  KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning "
354  "TASK_NOT_PUSHED for task %p\n",
355  gtid, taskdata));
356  return TASK_NOT_PUSHED;
357  }
358 
359  // Now that serialized tasks have returned, we can assume that we are not in
360  // immediate exec mode
361  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
362  if (UNLIKELY(!KMP_TASKING_ENABLED(task_team))) {
363  __kmp_enable_tasking(task_team, thread);
364  }
365  KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE);
366  KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL);
367 
368  // Find tasking deque specific to encountering thread
369  thread_data = &task_team->tt.tt_threads_data[tid];
370 
371  // No lock needed since only owner can allocate. If the task is hidden_helper,
372  // we don't need it either because we have initialized the dequeue for hidden
373  // helper thread data.
374  if (UNLIKELY(thread_data->td.td_deque == NULL)) {
375  __kmp_alloc_task_deque(thread, thread_data);
376  }
377 
378  int locked = 0;
379  // Check if deque is full
380  if (TCR_4(thread_data->td.td_deque_ntasks) >=
381  TASK_DEQUE_SIZE(thread_data->td)) {
382  if (__kmp_enable_task_throttling &&
383  __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
384  thread->th.th_current_task)) {
385  KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning "
386  "TASK_NOT_PUSHED for task %p\n",
387  gtid, taskdata));
388  return TASK_NOT_PUSHED;
389  } else {
390  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
391  locked = 1;
392  if (TCR_4(thread_data->td.td_deque_ntasks) >=
393  TASK_DEQUE_SIZE(thread_data->td)) {
394  // expand deque to push the task which is not allowed to execute
395  __kmp_realloc_task_deque(thread, thread_data);
396  }
397  }
398  }
399  // Lock the deque for the task push operation
400  if (!locked) {
401  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
402  // Need to recheck as we can get a proxy task from thread outside of OpenMP
403  if (TCR_4(thread_data->td.td_deque_ntasks) >=
404  TASK_DEQUE_SIZE(thread_data->td)) {
405  if (__kmp_enable_task_throttling &&
406  __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
407  thread->th.th_current_task)) {
408  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
409  KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; "
410  "returning TASK_NOT_PUSHED for task %p\n",
411  gtid, taskdata));
412  return TASK_NOT_PUSHED;
413  } else {
414  // expand deque to push the task which is not allowed to execute
415  __kmp_realloc_task_deque(thread, thread_data);
416  }
417  }
418  }
419  // Must have room since no thread can add tasks but calling thread
420  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) <
421  TASK_DEQUE_SIZE(thread_data->td));
422 
423  thread_data->td.td_deque[thread_data->td.td_deque_tail] =
424  taskdata; // Push taskdata
425  // Wrap index.
426  thread_data->td.td_deque_tail =
427  (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
428  TCW_4(thread_data->td.td_deque_ntasks,
429  TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count
430  KMP_FSYNC_RELEASING(thread->th.th_current_task); // releasing self
431  KMP_FSYNC_RELEASING(taskdata); // releasing child
432  KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
433  "task=%p ntasks=%d head=%u tail=%u\n",
434  gtid, taskdata, thread_data->td.td_deque_ntasks,
435  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
436 
437  auto hidden_helper = taskdata->td_flags.hidden_helper;
438 
439  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
440 
441  // Signal one worker thread to execute the task
442  if (UNLIKELY(hidden_helper)) {
443  // Wake hidden helper threads up if they're sleeping
444  __kmp_hidden_helper_worker_thread_signal();
445  }
446 
447  return TASK_SUCCESSFULLY_PUSHED;
448 }
449 
450 // __kmp_pop_current_task_from_thread: set up current task from called thread
451 // when team ends
452 //
453 // this_thr: thread structure to set current_task in.
454 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) {
455  KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d "
456  "this_thread=%p, curtask=%p, "
457  "curtask_parent=%p\n",
458  0, this_thr, this_thr->th.th_current_task,
459  this_thr->th.th_current_task->td_parent));
460 
461  this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent;
462 
463  KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d "
464  "this_thread=%p, curtask=%p, "
465  "curtask_parent=%p\n",
466  0, this_thr, this_thr->th.th_current_task,
467  this_thr->th.th_current_task->td_parent));
468 }
469 
470 // __kmp_push_current_task_to_thread: set up current task in called thread for a
471 // new team
472 //
473 // this_thr: thread structure to set up
474 // team: team for implicit task data
475 // tid: thread within team to set up
476 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team,
477  int tid) {
478  // current task of the thread is a parent of the new just created implicit
479  // tasks of new team
480  KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
481  "curtask=%p "
482  "parent_task=%p\n",
483  tid, this_thr, this_thr->th.th_current_task,
484  team->t.t_implicit_task_taskdata[tid].td_parent));
485 
486  KMP_DEBUG_ASSERT(this_thr != NULL);
487 
488  if (tid == 0) {
489  if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) {
490  team->t.t_implicit_task_taskdata[0].td_parent =
491  this_thr->th.th_current_task;
492  this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0];
493  }
494  } else {
495  team->t.t_implicit_task_taskdata[tid].td_parent =
496  team->t.t_implicit_task_taskdata[0].td_parent;
497  this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid];
498  }
499 
500  KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
501  "curtask=%p "
502  "parent_task=%p\n",
503  tid, this_thr, this_thr->th.th_current_task,
504  team->t.t_implicit_task_taskdata[tid].td_parent));
505 }
506 
507 // __kmp_task_start: bookkeeping for a task starting execution
508 //
509 // GTID: global thread id of calling thread
510 // task: task starting execution
511 // current_task: task suspending
512 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task,
513  kmp_taskdata_t *current_task) {
514  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
515  kmp_info_t *thread = __kmp_threads[gtid];
516 
517  KA_TRACE(10,
518  ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
519  gtid, taskdata, current_task));
520 
521  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
522 
523  // mark currently executing task as suspended
524  // TODO: GEH - make sure root team implicit task is initialized properly.
525  // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 );
526  current_task->td_flags.executing = 0;
527 
528 // Add task to stack if tied
529 #ifdef BUILD_TIED_TASK_STACK
530  if (taskdata->td_flags.tiedness == TASK_TIED) {
531  __kmp_push_task_stack(gtid, thread, taskdata);
532  }
533 #endif /* BUILD_TIED_TASK_STACK */
534 
535  // mark starting task as executing and as current task
536  thread->th.th_current_task = taskdata;
537 
538  KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 ||
539  taskdata->td_flags.tiedness == TASK_UNTIED);
540  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 ||
541  taskdata->td_flags.tiedness == TASK_UNTIED);
542  taskdata->td_flags.started = 1;
543  taskdata->td_flags.executing = 1;
544  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
545  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
546 
547  // GEH TODO: shouldn't we pass some sort of location identifier here?
548  // APT: yes, we will pass location here.
549  // need to store current thread state (in a thread or taskdata structure)
550  // before setting work_state, otherwise wrong state is set after end of task
551 
552  KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata));
553 
554  return;
555 }
556 
557 #if OMPT_SUPPORT
558 //------------------------------------------------------------------------------
559 // __ompt_task_init:
560 // Initialize OMPT fields maintained by a task. This will only be called after
561 // ompt_start_tool, so we already know whether ompt is enabled or not.
562 
563 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) {
564  // The calls to __ompt_task_init already have the ompt_enabled condition.
565  task->ompt_task_info.task_data.value = 0;
566  task->ompt_task_info.frame.exit_frame = ompt_data_none;
567  task->ompt_task_info.frame.enter_frame = ompt_data_none;
568  task->ompt_task_info.frame.exit_frame_flags =
569  ompt_frame_runtime | ompt_frame_framepointer;
570  task->ompt_task_info.frame.enter_frame_flags =
571  ompt_frame_runtime | ompt_frame_framepointer;
572 }
573 
574 // __ompt_task_start:
575 // Build and trigger task-begin event
576 static inline void __ompt_task_start(kmp_task_t *task,
577  kmp_taskdata_t *current_task,
578  kmp_int32 gtid) {
579  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
580  ompt_task_status_t status = ompt_task_switch;
581  if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) {
582  status = ompt_task_yield;
583  __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0;
584  }
585  /* let OMPT know that we're about to run this task */
586  if (ompt_enabled.ompt_callback_task_schedule) {
587  ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
588  &(current_task->ompt_task_info.task_data), status,
589  &(taskdata->ompt_task_info.task_data));
590  }
591  taskdata->ompt_task_info.scheduling_parent = current_task;
592 }
593 
594 // __ompt_task_finish:
595 // Build and trigger final task-schedule event
596 static inline void __ompt_task_finish(kmp_task_t *task,
597  kmp_taskdata_t *resumed_task,
598  ompt_task_status_t status) {
599  if (ompt_enabled.ompt_callback_task_schedule) {
600  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
601  if (__kmp_omp_cancellation && taskdata->td_taskgroup &&
602  taskdata->td_taskgroup->cancel_request == cancel_taskgroup) {
603  status = ompt_task_cancel;
604  }
605 
606  /* let OMPT know that we're returning to the callee task */
607  ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
608  &(taskdata->ompt_task_info.task_data), status,
609  (resumed_task ? &(resumed_task->ompt_task_info.task_data) : NULL));
610  }
611 }
612 #endif
613 
614 template <bool ompt>
615 static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid,
616  kmp_task_t *task,
617  void *frame_address,
618  void *return_address) {
619  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
620  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
621 
622  KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
623  "current_task=%p\n",
624  gtid, loc_ref, taskdata, current_task));
625 
626  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
627  // untied task needs to increment counter so that the task structure is not
628  // freed prematurely
629  kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count);
630  KMP_DEBUG_USE_VAR(counter);
631  KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "
632  "incremented for task %p\n",
633  gtid, counter, taskdata));
634  }
635 
636  taskdata->td_flags.task_serial =
637  1; // Execute this task immediately, not deferred.
638  __kmp_task_start(gtid, task, current_task);
639 
640 #if OMPT_SUPPORT
641  if (ompt) {
642  if (current_task->ompt_task_info.frame.enter_frame.ptr == NULL) {
643  current_task->ompt_task_info.frame.enter_frame.ptr =
644  taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address;
645  current_task->ompt_task_info.frame.enter_frame_flags =
646  taskdata->ompt_task_info.frame.exit_frame_flags =
647  ompt_frame_application | ompt_frame_framepointer;
648  }
649  if (ompt_enabled.ompt_callback_task_create) {
650  ompt_task_info_t *parent_info = &(current_task->ompt_task_info);
651  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
652  &(parent_info->task_data), &(parent_info->frame),
653  &(taskdata->ompt_task_info.task_data),
654  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0,
655  return_address);
656  }
657  __ompt_task_start(task, current_task, gtid);
658  }
659 #endif // OMPT_SUPPORT
660 
661  KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid,
662  loc_ref, taskdata));
663 }
664 
665 #if OMPT_SUPPORT
666 OMPT_NOINLINE
667 static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
668  kmp_task_t *task,
669  void *frame_address,
670  void *return_address) {
671  __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address,
672  return_address);
673 }
674 #endif // OMPT_SUPPORT
675 
676 // __kmpc_omp_task_begin_if0: report that a given serialized task has started
677 // execution
678 //
679 // loc_ref: source location information; points to beginning of task block.
680 // gtid: global thread number.
681 // task: task thunk for the started task.
682 void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid,
683  kmp_task_t *task) {
684 #if OMPT_SUPPORT
685  if (UNLIKELY(ompt_enabled.enabled)) {
686  OMPT_STORE_RETURN_ADDRESS(gtid);
687  __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task,
688  OMPT_GET_FRAME_ADDRESS(1),
689  OMPT_LOAD_RETURN_ADDRESS(gtid));
690  return;
691  }
692 #endif
693  __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL);
694 }
695 
696 #ifdef TASK_UNUSED
697 // __kmpc_omp_task_begin: report that a given task has started execution
698 // NEVER GENERATED BY COMPILER, DEPRECATED!!!
699 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) {
700  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
701 
702  KA_TRACE(
703  10,
704  ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
705  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task));
706 
707  __kmp_task_start(gtid, task, current_task);
708 
709  KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid,
710  loc_ref, KMP_TASK_TO_TASKDATA(task)));
711  return;
712 }
713 #endif // TASK_UNUSED
714 
715 // __kmp_free_task: free the current task space and the space for shareds
716 //
717 // gtid: Global thread ID of calling thread
718 // taskdata: task to free
719 // thread: thread data structure of caller
720 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
721  kmp_info_t *thread) {
722  KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid,
723  taskdata));
724 
725  // Check to make sure all flags and counters have the correct values
726  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
727  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0);
728  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1);
729  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
730  KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 ||
731  taskdata->td_flags.task_serial == 1);
732  KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0);
733 
734  taskdata->td_flags.freed = 1;
735 // deallocate the taskdata and shared variable blocks associated with this task
736 #if USE_FAST_MEMORY
737  __kmp_fast_free(thread, taskdata);
738 #else /* ! USE_FAST_MEMORY */
739  __kmp_thread_free(thread, taskdata);
740 #endif
741  KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
742 }
743 
744 // __kmp_free_task_and_ancestors: free the current task and ancestors without
745 // children
746 //
747 // gtid: Global thread ID of calling thread
748 // taskdata: task to free
749 // thread: thread data structure of caller
750 static void __kmp_free_task_and_ancestors(kmp_int32 gtid,
751  kmp_taskdata_t *taskdata,
752  kmp_info_t *thread) {
753  // Proxy tasks must always be allowed to free their parents
754  // because they can be run in background even in serial mode.
755  kmp_int32 team_serial =
756  (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) &&
757  !taskdata->td_flags.proxy;
758  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
759 
760  kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1;
761  KMP_DEBUG_ASSERT(children >= 0);
762 
763  // Now, go up the ancestor tree to see if any ancestors can now be freed.
764  while (children == 0) {
765  kmp_taskdata_t *parent_taskdata = taskdata->td_parent;
766 
767  KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
768  "and freeing itself\n",
769  gtid, taskdata));
770 
771  // --- Deallocate my ancestor task ---
772  __kmp_free_task(gtid, taskdata, thread);
773 
774  taskdata = parent_taskdata;
775 
776  if (team_serial)
777  return;
778  // Stop checking ancestors at implicit task instead of walking up ancestor
779  // tree to avoid premature deallocation of ancestors.
780  if (taskdata->td_flags.tasktype == TASK_IMPLICIT) {
781  if (taskdata->td_dephash) { // do we need to cleanup dephash?
782  int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks);
783  kmp_tasking_flags_t flags_old = taskdata->td_flags;
784  if (children == 0 && flags_old.complete == 1) {
785  kmp_tasking_flags_t flags_new = flags_old;
786  flags_new.complete = 0;
787  if (KMP_COMPARE_AND_STORE_ACQ32(
788  RCAST(kmp_int32 *, &taskdata->td_flags),
789  *RCAST(kmp_int32 *, &flags_old),
790  *RCAST(kmp_int32 *, &flags_new))) {
791  KA_TRACE(100, ("__kmp_free_task_and_ancestors: T#%d cleans "
792  "dephash of implicit task %p\n",
793  gtid, taskdata));
794  // cleanup dephash of finished implicit task
795  __kmp_dephash_free_entries(thread, taskdata->td_dephash);
796  }
797  }
798  }
799  return;
800  }
801  // Predecrement simulated by "- 1" calculation
802  children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1;
803  KMP_DEBUG_ASSERT(children >= 0);
804  }
805 
806  KA_TRACE(
807  20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
808  "not freeing it yet\n",
809  gtid, taskdata, children));
810 }
811 
812 // __kmp_task_finish: bookkeeping to do when a task finishes execution
813 //
814 // gtid: global thread ID for calling thread
815 // task: task to be finished
816 // resumed_task: task to be resumed. (may be NULL if task is serialized)
817 //
818 // template<ompt>: effectively ompt_enabled.enabled!=0
819 // the version with ompt=false is inlined, allowing to optimize away all ompt
820 // code in this case
821 template <bool ompt>
822 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
823  kmp_taskdata_t *resumed_task) {
824  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
825  kmp_info_t *thread = __kmp_threads[gtid];
826  kmp_task_team_t *task_team =
827  thread->th.th_task_team; // might be NULL for serial teams...
828  kmp_int32 children = 0;
829 
830  KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "
831  "task %p\n",
832  gtid, taskdata, resumed_task));
833 
834  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
835 
836 // Pop task from stack if tied
837 #ifdef BUILD_TIED_TASK_STACK
838  if (taskdata->td_flags.tiedness == TASK_TIED) {
839  __kmp_pop_task_stack(gtid, thread, taskdata);
840  }
841 #endif /* BUILD_TIED_TASK_STACK */
842 
843  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
844  // untied task needs to check the counter so that the task structure is not
845  // freed prematurely
846  kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1;
847  KA_TRACE(
848  20,
849  ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n",
850  gtid, counter, taskdata));
851  if (counter > 0) {
852  // untied task is not done, to be continued possibly by other thread, do
853  // not free it now
854  if (resumed_task == NULL) {
855  KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial);
856  resumed_task = taskdata->td_parent; // In a serialized task, the resumed
857  // task is the parent
858  }
859  thread->th.th_current_task = resumed_task; // restore current_task
860  resumed_task->td_flags.executing = 1; // resume previous task
861  KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, "
862  "resuming task %p\n",
863  gtid, taskdata, resumed_task));
864  return;
865  }
866  }
867 
868  // bookkeeping for resuming task:
869  // GEH - note tasking_ser => task_serial
870  KMP_DEBUG_ASSERT(
871  (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==
872  taskdata->td_flags.task_serial);
873  if (taskdata->td_flags.task_serial) {
874  if (resumed_task == NULL) {
875  resumed_task = taskdata->td_parent; // In a serialized task, the resumed
876  // task is the parent
877  }
878  } else {
879  KMP_DEBUG_ASSERT(resumed_task !=
880  NULL); // verify that resumed task is passed as argument
881  }
882 
883  /* If the tasks' destructor thunk flag has been set, we need to invoke the
884  destructor thunk that has been generated by the compiler. The code is
885  placed here, since at this point other tasks might have been released
886  hence overlapping the destructor invocations with some other work in the
887  released tasks. The OpenMP spec is not specific on when the destructors
888  are invoked, so we should be free to choose. */
889  if (UNLIKELY(taskdata->td_flags.destructors_thunk)) {
890  kmp_routine_entry_t destr_thunk = task->data1.destructors;
891  KMP_ASSERT(destr_thunk);
892  destr_thunk(gtid, task);
893  }
894 
895  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
896  KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1);
897  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
898 
899  bool detach = false;
900  if (UNLIKELY(taskdata->td_flags.detachable == TASK_DETACHABLE)) {
901  if (taskdata->td_allow_completion_event.type ==
902  KMP_EVENT_ALLOW_COMPLETION) {
903  // event hasn't been fulfilled yet. Try to detach task.
904  __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid);
905  if (taskdata->td_allow_completion_event.type ==
906  KMP_EVENT_ALLOW_COMPLETION) {
907  // task finished execution
908  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1);
909  taskdata->td_flags.executing = 0; // suspend the finishing task
910 
911 #if OMPT_SUPPORT
912  // For a detached task, which is not completed, we switch back
913  // the omp_fulfill_event signals completion
914  // locking is necessary to avoid a race with ompt_task_late_fulfill
915  if (ompt)
916  __ompt_task_finish(task, resumed_task, ompt_task_detach);
917 #endif
918 
919  // no access to taskdata after this point!
920  // __kmp_fulfill_event might free taskdata at any time from now
921 
922  taskdata->td_flags.proxy = TASK_PROXY; // proxify!
923  detach = true;
924  }
925  __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid);
926  }
927  }
928 
929  if (!detach) {
930  taskdata->td_flags.complete = 1; // mark the task as completed
931 
932 #if OMPT_SUPPORT
933  // This is not a detached task, we are done here
934  if (ompt)
935  __ompt_task_finish(task, resumed_task, ompt_task_complete);
936 #endif
937 
938  // Only need to keep track of count if team parallel and tasking not
939  // serialized, or task is detachable and event has already been fulfilled
940  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) ||
941  taskdata->td_flags.detachable == TASK_DETACHABLE ||
942  taskdata->td_flags.hidden_helper) {
943  __kmp_release_deps(gtid, taskdata);
944  // Predecrement simulated by "- 1" calculation
945  children =
946  KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1;
947  KMP_DEBUG_ASSERT(children >= 0);
948  if (taskdata->td_taskgroup)
949  KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
950  } else if (task_team && (task_team->tt.tt_found_proxy_tasks ||
951  task_team->tt.tt_hidden_helper_task_encountered)) {
952  // if we found proxy or hidden helper tasks there could exist a dependency
953  // chain with the proxy task as origin
954  __kmp_release_deps(gtid, taskdata);
955  }
956  // td_flags.executing must be marked as 0 after __kmp_release_deps has been
957  // called. Othertwise, if a task is executed immediately from the
958  // release_deps code, the flag will be reset to 1 again by this same
959  // function
960  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1);
961  taskdata->td_flags.executing = 0; // suspend the finishing task
962  }
963 
964  KA_TRACE(
965  20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
966  gtid, taskdata, children));
967 
968  // Free this task and then ancestor tasks if they have no children.
969  // Restore th_current_task first as suggested by John:
970  // johnmc: if an asynchronous inquiry peers into the runtime system
971  // it doesn't see the freed task as the current task.
972  thread->th.th_current_task = resumed_task;
973  if (!detach)
974  __kmp_free_task_and_ancestors(gtid, taskdata, thread);
975 
976  // TODO: GEH - make sure root team implicit task is initialized properly.
977  // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 );
978  resumed_task->td_flags.executing = 1; // resume previous task
979 
980  KA_TRACE(
981  10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
982  gtid, taskdata, resumed_task));
983 
984  return;
985 }
986 
987 template <bool ompt>
988 static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref,
989  kmp_int32 gtid,
990  kmp_task_t *task) {
991  KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
992  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
993  KMP_DEBUG_ASSERT(gtid >= 0);
994  // this routine will provide task to resume
995  __kmp_task_finish<ompt>(gtid, task, NULL);
996 
997  KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
998  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
999 
1000 #if OMPT_SUPPORT
1001  if (ompt) {
1002  ompt_frame_t *ompt_frame;
1003  __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1004  ompt_frame->enter_frame = ompt_data_none;
1005  ompt_frame->enter_frame_flags =
1006  ompt_frame_runtime | ompt_frame_framepointer;
1007  }
1008 #endif
1009 
1010  return;
1011 }
1012 
1013 #if OMPT_SUPPORT
1014 OMPT_NOINLINE
1015 void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
1016  kmp_task_t *task) {
1017  __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task);
1018 }
1019 #endif // OMPT_SUPPORT
1020 
1021 // __kmpc_omp_task_complete_if0: report that a task has completed execution
1022 //
1023 // loc_ref: source location information; points to end of task block.
1024 // gtid: global thread number.
1025 // task: task thunk for the completed task.
1026 void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid,
1027  kmp_task_t *task) {
1028 #if OMPT_SUPPORT
1029  if (UNLIKELY(ompt_enabled.enabled)) {
1030  __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task);
1031  return;
1032  }
1033 #endif
1034  __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task);
1035 }
1036 
1037 #ifdef TASK_UNUSED
1038 // __kmpc_omp_task_complete: report that a task has completed execution
1039 // NEVER GENERATED BY COMPILER, DEPRECATED!!!
1040 void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid,
1041  kmp_task_t *task) {
1042  KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid,
1043  loc_ref, KMP_TASK_TO_TASKDATA(task)));
1044 
1045  __kmp_task_finish<false>(gtid, task,
1046  NULL); // Not sure how to find task to resume
1047 
1048  KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid,
1049  loc_ref, KMP_TASK_TO_TASKDATA(task)));
1050  return;
1051 }
1052 #endif // TASK_UNUSED
1053 
1054 // __kmp_init_implicit_task: Initialize the appropriate fields in the implicit
1055 // task for a given thread
1056 //
1057 // loc_ref: reference to source location of parallel region
1058 // this_thr: thread data structure corresponding to implicit task
1059 // team: team for this_thr
1060 // tid: thread id of given thread within team
1061 // set_curr_task: TRUE if need to push current task to thread
1062 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to
1063 // have already been done elsewhere.
1064 // TODO: Get better loc_ref. Value passed in may be NULL
1065 void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
1066  kmp_team_t *team, int tid, int set_curr_task) {
1067  kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid];
1068 
1069  KF_TRACE(
1070  10,
1071  ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
1072  tid, team, task, set_curr_task ? "TRUE" : "FALSE"));
1073 
1074  task->td_task_id = KMP_GEN_TASK_ID();
1075  task->td_team = team;
1076  // task->td_parent = NULL; // fix for CQ230101 (broken parent task info
1077  // in debugger)
1078  task->td_ident = loc_ref;
1079  task->td_taskwait_ident = NULL;
1080  task->td_taskwait_counter = 0;
1081  task->td_taskwait_thread = 0;
1082 
1083  task->td_flags.tiedness = TASK_TIED;
1084  task->td_flags.tasktype = TASK_IMPLICIT;
1085  task->td_flags.proxy = TASK_FULL;
1086 
1087  // All implicit tasks are executed immediately, not deferred
1088  task->td_flags.task_serial = 1;
1089  task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1090  task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1091 
1092  task->td_flags.started = 1;
1093  task->td_flags.executing = 1;
1094  task->td_flags.complete = 0;
1095  task->td_flags.freed = 0;
1096 
1097  task->td_depnode = NULL;
1098  task->td_last_tied = task;
1099  task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED;
1100 
1101  if (set_curr_task) { // only do this init first time thread is created
1102  KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0);
1103  // Not used: don't need to deallocate implicit task
1104  KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0);
1105  task->td_taskgroup = NULL; // An implicit task does not have taskgroup
1106  task->td_dephash = NULL;
1107  __kmp_push_current_task_to_thread(this_thr, team, tid);
1108  } else {
1109  KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
1110  KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
1111  }
1112 
1113 #if OMPT_SUPPORT
1114  if (UNLIKELY(ompt_enabled.enabled))
1115  __ompt_task_init(task, tid);
1116 #endif
1117 
1118  KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid,
1119  team, task));
1120 }
1121 
1122 // __kmp_finish_implicit_task: Release resources associated to implicit tasks
1123 // at the end of parallel regions. Some resources are kept for reuse in the next
1124 // parallel region.
1125 //
1126 // thread: thread data structure corresponding to implicit task
1127 void __kmp_finish_implicit_task(kmp_info_t *thread) {
1128  kmp_taskdata_t *task = thread->th.th_current_task;
1129  if (task->td_dephash) {
1130  int children;
1131  task->td_flags.complete = 1;
1132  children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks);
1133  kmp_tasking_flags_t flags_old = task->td_flags;
1134  if (children == 0 && flags_old.complete == 1) {
1135  kmp_tasking_flags_t flags_new = flags_old;
1136  flags_new.complete = 0;
1137  if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags),
1138  *RCAST(kmp_int32 *, &flags_old),
1139  *RCAST(kmp_int32 *, &flags_new))) {
1140  KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans "
1141  "dephash of implicit task %p\n",
1142  thread->th.th_info.ds.ds_gtid, task));
1143  __kmp_dephash_free_entries(thread, task->td_dephash);
1144  }
1145  }
1146  }
1147 }
1148 
1149 // __kmp_free_implicit_task: Release resources associated to implicit tasks
1150 // when these are destroyed regions
1151 //
1152 // thread: thread data structure corresponding to implicit task
1153 void __kmp_free_implicit_task(kmp_info_t *thread) {
1154  kmp_taskdata_t *task = thread->th.th_current_task;
1155  if (task && task->td_dephash) {
1156  __kmp_dephash_free(thread, task->td_dephash);
1157  task->td_dephash = NULL;
1158  }
1159 }
1160 
1161 // Round up a size to a power of two specified by val: Used to insert padding
1162 // between structures co-allocated using a single malloc() call
1163 static size_t __kmp_round_up_to_val(size_t size, size_t val) {
1164  if (size & (val - 1)) {
1165  size &= ~(val - 1);
1166  if (size <= KMP_SIZE_T_MAX - val) {
1167  size += val; // Round up if there is no overflow.
1168  }
1169  }
1170  return size;
1171 } // __kmp_round_up_to_va
1172 
1173 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task
1174 //
1175 // loc_ref: source location information
1176 // gtid: global thread number.
1177 // flags: include tiedness & task type (explicit vs. implicit) of the ''new''
1178 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine.
1179 // sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including
1180 // private vars accessed in task.
1181 // sizeof_shareds: Size in bytes of array of pointers to shared vars accessed
1182 // in task.
1183 // task_entry: Pointer to task code entry point generated by compiler.
1184 // returns: a pointer to the allocated kmp_task_t structure (task).
1185 kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1186  kmp_tasking_flags_t *flags,
1187  size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1188  kmp_routine_entry_t task_entry) {
1189  kmp_task_t *task;
1190  kmp_taskdata_t *taskdata;
1191  kmp_info_t *thread = __kmp_threads[gtid];
1192  kmp_info_t *encountering_thread = thread;
1193  kmp_team_t *team = thread->th.th_team;
1194  kmp_taskdata_t *parent_task = thread->th.th_current_task;
1195  size_t shareds_offset;
1196 
1197  if (UNLIKELY(!TCR_4(__kmp_init_middle)))
1198  __kmp_middle_initialize();
1199 
1200  if (flags->hidden_helper) {
1201  if (__kmp_enable_hidden_helper) {
1202  if (!TCR_4(__kmp_init_hidden_helper))
1203  __kmp_hidden_helper_initialize();
1204 
1205  // For a hidden helper task encountered by a regular thread, we will push
1206  // the task to the (gtid%__kmp_hidden_helper_threads_num)-th hidden helper
1207  // thread.
1208  if (!KMP_HIDDEN_HELPER_THREAD(gtid)) {
1209  thread = __kmp_threads[KMP_GTID_TO_SHADOW_GTID(gtid)];
1210  // We don't change the parent-child relation for hidden helper task as
1211  // we need that to do per-task-region synchronization.
1212  }
1213  } else {
1214  // If the hidden helper task is not enabled, reset the flag to FALSE.
1215  flags->hidden_helper = FALSE;
1216  }
1217  }
1218 
1219  KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
1220  "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1221  gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
1222  sizeof_shareds, task_entry));
1223 
1224  KMP_DEBUG_ASSERT(parent_task);
1225  if (parent_task->td_flags.final) {
1226  if (flags->merged_if0) {
1227  }
1228  flags->final = 1;
1229  }
1230 
1231  if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) {
1232  // Untied task encountered causes the TSC algorithm to check entire deque of
1233  // the victim thread. If no untied task encountered, then checking the head
1234  // of the deque should be enough.
1235  KMP_CHECK_UPDATE(
1236  encountering_thread->th.th_task_team->tt.tt_untied_task_encountered, 1);
1237  }
1238 
1239  // Detachable tasks are not proxy tasks yet but could be in the future. Doing
1240  // the tasking setup
1241  // when that happens is too late.
1242  if (UNLIKELY(flags->proxy == TASK_PROXY ||
1243  flags->detachable == TASK_DETACHABLE || flags->hidden_helper)) {
1244  if (flags->proxy == TASK_PROXY) {
1245  flags->tiedness = TASK_UNTIED;
1246  flags->merged_if0 = 1;
1247  }
1248  /* are we running in a sequential parallel or tskm_immediate_exec... we need
1249  tasking support enabled */
1250  if ((encountering_thread->th.th_task_team) == NULL) {
1251  /* This should only happen if the team is serialized
1252  setup a task team and propagate it to the thread */
1253  KMP_DEBUG_ASSERT(team->t.t_serialized);
1254  KA_TRACE(30,
1255  ("T#%d creating task team in __kmp_task_alloc for proxy task\n",
1256  gtid));
1257  __kmp_task_team_setup(
1258  encountering_thread, team,
1259  1); // 1 indicates setup the current team regardless of nthreads
1260  encountering_thread->th.th_task_team =
1261  team->t.t_task_team[encountering_thread->th.th_task_state];
1262  }
1263  kmp_task_team_t *task_team = encountering_thread->th.th_task_team;
1264 
1265  /* tasking must be enabled now as the task might not be pushed */
1266  if (!KMP_TASKING_ENABLED(task_team)) {
1267  KA_TRACE(
1268  30,
1269  ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid));
1270  __kmp_enable_tasking(task_team, encountering_thread);
1271  kmp_int32 tid = encountering_thread->th.th_info.ds.ds_tid;
1272  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
1273  // No lock needed since only owner can allocate
1274  if (thread_data->td.td_deque == NULL) {
1275  __kmp_alloc_task_deque(encountering_thread, thread_data);
1276  }
1277  }
1278 
1279  if ((flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE) &&
1280  task_team->tt.tt_found_proxy_tasks == FALSE)
1281  TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE);
1282  if (flags->hidden_helper &&
1283  task_team->tt.tt_hidden_helper_task_encountered == FALSE)
1284  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, TRUE);
1285  }
1286 
1287  // Calculate shared structure offset including padding after kmp_task_t struct
1288  // to align pointers in shared struct
1289  shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
1290  shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *));
1291 
1292  // Allocate a kmp_taskdata_t block and a kmp_task_t block.
1293  KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid,
1294  shareds_offset));
1295  KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid,
1296  sizeof_shareds));
1297 
1298  // Avoid double allocation here by combining shareds with taskdata
1299 #if USE_FAST_MEMORY
1300  taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(
1301  encountering_thread, shareds_offset + sizeof_shareds);
1302 #else /* ! USE_FAST_MEMORY */
1303  taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(
1304  encountering_thread, shareds_offset + sizeof_shareds);
1305 #endif /* USE_FAST_MEMORY */
1306 
1307  task = KMP_TASKDATA_TO_TASK(taskdata);
1308 
1309 // Make sure task & taskdata are aligned appropriately
1310 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD
1311  KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0);
1312  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0);
1313 #else
1314  KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0);
1315  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0);
1316 #endif
1317  if (sizeof_shareds > 0) {
1318  // Avoid double allocation here by combining shareds with taskdata
1319  task->shareds = &((char *)taskdata)[shareds_offset];
1320  // Make sure shareds struct is aligned to pointer size
1321  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
1322  0);
1323  } else {
1324  task->shareds = NULL;
1325  }
1326  task->routine = task_entry;
1327  task->part_id = 0; // AC: Always start with 0 part id
1328 
1329  taskdata->td_task_id = KMP_GEN_TASK_ID();
1330  taskdata->td_team = thread->th.th_team;
1331  taskdata->td_alloc_thread = encountering_thread;
1332  taskdata->td_parent = parent_task;
1333  taskdata->td_level = parent_task->td_level + 1; // increment nesting level
1334  KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
1335  taskdata->td_ident = loc_ref;
1336  taskdata->td_taskwait_ident = NULL;
1337  taskdata->td_taskwait_counter = 0;
1338  taskdata->td_taskwait_thread = 0;
1339  KMP_DEBUG_ASSERT(taskdata->td_parent != NULL);
1340  // avoid copying icvs for proxy tasks
1341  if (flags->proxy == TASK_FULL)
1342  copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs);
1343 
1344  taskdata->td_flags = *flags;
1345  taskdata->encountering_gtid = gtid;
1346  taskdata->td_task_team = thread->th.th_task_team;
1347  taskdata->td_size_alloc = shareds_offset + sizeof_shareds;
1348  taskdata->td_flags.tasktype = TASK_EXPLICIT;
1349 
1350  // GEH - TODO: fix this to copy parent task's value of tasking_ser flag
1351  taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1352 
1353  // GEH - TODO: fix this to copy parent task's value of team_serial flag
1354  taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1355 
1356  // GEH - Note we serialize the task if the team is serialized to make sure
1357  // implicit parallel region tasks are not left until program termination to
1358  // execute. Also, it helps locality to execute immediately.
1359 
1360  taskdata->td_flags.task_serial =
1361  (parent_task->td_flags.final || taskdata->td_flags.team_serial ||
1362  taskdata->td_flags.tasking_ser || flags->merged_if0);
1363 
1364  taskdata->td_flags.started = 0;
1365  taskdata->td_flags.executing = 0;
1366  taskdata->td_flags.complete = 0;
1367  taskdata->td_flags.freed = 0;
1368 
1369  KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
1370  // start at one because counts current task and children
1371  KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
1372  taskdata->td_taskgroup =
1373  parent_task->td_taskgroup; // task inherits taskgroup from the parent task
1374  taskdata->td_dephash = NULL;
1375  taskdata->td_depnode = NULL;
1376  if (flags->tiedness == TASK_UNTIED)
1377  taskdata->td_last_tied = NULL; // will be set when the task is scheduled
1378  else
1379  taskdata->td_last_tied = taskdata;
1380  taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED;
1381 #if OMPT_SUPPORT
1382  if (UNLIKELY(ompt_enabled.enabled))
1383  __ompt_task_init(taskdata, gtid);
1384 #endif
1385  // Only need to keep track of child task counts if team parallel and tasking
1386  // not serialized or if it is a proxy or detachable or hidden helper task
1387  if (flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE ||
1388  flags->hidden_helper ||
1389  !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
1390  KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
1391  if (parent_task->td_taskgroup)
1392  KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
1393  // Only need to keep track of allocated child tasks for explicit tasks since
1394  // implicit not deallocated
1395  if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) {
1396  KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks);
1397  }
1398  if (flags->hidden_helper) {
1399  taskdata->td_flags.task_serial = FALSE;
1400  // Increment the number of hidden helper tasks to be executed
1401  KMP_ATOMIC_INC(&__kmp_unexecuted_hidden_helper_tasks);
1402  }
1403  }
1404 
1405  KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
1406  gtid, taskdata, taskdata->td_parent));
1407 
1408  return task;
1409 }
1410 
1411 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1412  kmp_int32 flags, size_t sizeof_kmp_task_t,
1413  size_t sizeof_shareds,
1414  kmp_routine_entry_t task_entry) {
1415  kmp_task_t *retval;
1416  kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
1417  __kmp_assert_valid_gtid(gtid);
1418  input_flags->native = FALSE;
1419  // __kmp_task_alloc() sets up all other runtime flags
1420  KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) "
1421  "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1422  gtid, loc_ref, input_flags->tiedness ? "tied " : "untied",
1423  input_flags->proxy ? "proxy" : "",
1424  input_flags->detachable ? "detachable" : "", sizeof_kmp_task_t,
1425  sizeof_shareds, task_entry));
1426 
1427  retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t,
1428  sizeof_shareds, task_entry);
1429 
1430  KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval));
1431 
1432  return retval;
1433 }
1434 
1435 kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1436  kmp_int32 flags,
1437  size_t sizeof_kmp_task_t,
1438  size_t sizeof_shareds,
1439  kmp_routine_entry_t task_entry,
1440  kmp_int64 device_id) {
1441  if (__kmp_enable_hidden_helper) {
1442  auto &input_flags = reinterpret_cast<kmp_tasking_flags_t &>(flags);
1443  input_flags.hidden_helper = TRUE;
1444  }
1445 
1446  return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t,
1447  sizeof_shareds, task_entry);
1448 }
1449 
1463 kmp_int32
1465  kmp_task_t *new_task, kmp_int32 naffins,
1466  kmp_task_affinity_info_t *affin_list) {
1467  return 0;
1468 }
1469 
1470 // __kmp_invoke_task: invoke the specified task
1471 //
1472 // gtid: global thread ID of caller
1473 // task: the task to invoke
1474 // current_task: the task to resume after task invocation
1475 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
1476  kmp_taskdata_t *current_task) {
1477  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
1478  kmp_info_t *thread;
1479  int discard = 0 /* false */;
1480  KA_TRACE(
1481  30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
1482  gtid, taskdata, current_task));
1483  KMP_DEBUG_ASSERT(task);
1484  if (UNLIKELY(taskdata->td_flags.proxy == TASK_PROXY &&
1485  taskdata->td_flags.complete == 1)) {
1486  // This is a proxy task that was already completed but it needs to run
1487  // its bottom-half finish
1488  KA_TRACE(
1489  30,
1490  ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n",
1491  gtid, taskdata));
1492 
1493  __kmp_bottom_half_finish_proxy(gtid, task);
1494 
1495  KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for "
1496  "proxy task %p, resuming task %p\n",
1497  gtid, taskdata, current_task));
1498 
1499  return;
1500  }
1501 
1502 #if OMPT_SUPPORT
1503  // For untied tasks, the first task executed only calls __kmpc_omp_task and
1504  // does not execute code.
1505  ompt_thread_info_t oldInfo;
1506  if (UNLIKELY(ompt_enabled.enabled)) {
1507  // Store the threads states and restore them after the task
1508  thread = __kmp_threads[gtid];
1509  oldInfo = thread->th.ompt_thread_info;
1510  thread->th.ompt_thread_info.wait_id = 0;
1511  thread->th.ompt_thread_info.state = (thread->th.th_team_serialized)
1512  ? ompt_state_work_serial
1513  : ompt_state_work_parallel;
1514  taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1515  }
1516 #endif
1517 
1518  // Decreament the counter of hidden helper tasks to be executed
1519  if (taskdata->td_flags.hidden_helper) {
1520  // Hidden helper tasks can only be executed by hidden helper threads
1521  KMP_ASSERT(KMP_HIDDEN_HELPER_THREAD(gtid));
1522  KMP_ATOMIC_DEC(&__kmp_unexecuted_hidden_helper_tasks);
1523  }
1524 
1525  // Proxy tasks are not handled by the runtime
1526  if (taskdata->td_flags.proxy != TASK_PROXY) {
1527  __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded
1528  }
1529 
1530  // TODO: cancel tasks if the parallel region has also been cancelled
1531  // TODO: check if this sequence can be hoisted above __kmp_task_start
1532  // if cancellation has been enabled for this run ...
1533  if (UNLIKELY(__kmp_omp_cancellation)) {
1534  thread = __kmp_threads[gtid];
1535  kmp_team_t *this_team = thread->th.th_team;
1536  kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
1537  if ((taskgroup && taskgroup->cancel_request) ||
1538  (this_team->t.t_cancel_request == cancel_parallel)) {
1539 #if OMPT_SUPPORT && OMPT_OPTIONAL
1540  ompt_data_t *task_data;
1541  if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) {
1542  __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
1543  ompt_callbacks.ompt_callback(ompt_callback_cancel)(
1544  task_data,
1545  ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup
1546  : ompt_cancel_parallel) |
1547  ompt_cancel_discarded_task,
1548  NULL);
1549  }
1550 #endif
1551  KMP_COUNT_BLOCK(TASK_cancelled);
1552  // this task belongs to a task group and we need to cancel it
1553  discard = 1 /* true */;
1554  }
1555  }
1556 
1557  // Invoke the task routine and pass in relevant data.
1558  // Thunks generated by gcc take a different argument list.
1559  if (!discard) {
1560  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
1561  taskdata->td_last_tied = current_task->td_last_tied;
1562  KMP_DEBUG_ASSERT(taskdata->td_last_tied);
1563  }
1564 #if KMP_STATS_ENABLED
1565  KMP_COUNT_BLOCK(TASK_executed);
1566  switch (KMP_GET_THREAD_STATE()) {
1567  case FORK_JOIN_BARRIER:
1568  KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar);
1569  break;
1570  case PLAIN_BARRIER:
1571  KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar);
1572  break;
1573  case TASKYIELD:
1574  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield);
1575  break;
1576  case TASKWAIT:
1577  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait);
1578  break;
1579  case TASKGROUP:
1580  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup);
1581  break;
1582  default:
1583  KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate);
1584  break;
1585  }
1586 #endif // KMP_STATS_ENABLED
1587 
1588 // OMPT task begin
1589 #if OMPT_SUPPORT
1590  if (UNLIKELY(ompt_enabled.enabled))
1591  __ompt_task_start(task, current_task, gtid);
1592 #endif
1593 
1594 #if OMPD_SUPPORT
1595  if (ompd_state & OMPD_ENABLE_BP)
1596  ompd_bp_task_begin();
1597 #endif
1598 
1599 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1600  kmp_uint64 cur_time;
1601  kmp_int32 kmp_itt_count_task =
1602  __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial &&
1603  current_task->td_flags.tasktype == TASK_IMPLICIT;
1604  if (kmp_itt_count_task) {
1605  thread = __kmp_threads[gtid];
1606  // Time outer level explicit task on barrier for adjusting imbalance time
1607  if (thread->th.th_bar_arrive_time)
1608  cur_time = __itt_get_timestamp();
1609  else
1610  kmp_itt_count_task = 0; // thread is not on a barrier - skip timing
1611  }
1612  KMP_FSYNC_ACQUIRED(taskdata); // acquired self (new task)
1613 #endif
1614 
1615 #ifdef KMP_GOMP_COMPAT
1616  if (taskdata->td_flags.native) {
1617  ((void (*)(void *))(*(task->routine)))(task->shareds);
1618  } else
1619 #endif /* KMP_GOMP_COMPAT */
1620  {
1621  (*(task->routine))(gtid, task);
1622  }
1623  KMP_POP_PARTITIONED_TIMER();
1624 
1625 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1626  if (kmp_itt_count_task) {
1627  // Barrier imbalance - adjust arrive time with the task duration
1628  thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time);
1629  }
1630  KMP_FSYNC_CANCEL(taskdata); // destroy self (just executed)
1631  KMP_FSYNC_RELEASING(taskdata->td_parent); // releasing parent
1632 #endif
1633  }
1634 
1635 #if OMPD_SUPPORT
1636  if (ompd_state & OMPD_ENABLE_BP)
1637  ompd_bp_task_end();
1638 #endif
1639 
1640  // Proxy tasks are not handled by the runtime
1641  if (taskdata->td_flags.proxy != TASK_PROXY) {
1642 #if OMPT_SUPPORT
1643  if (UNLIKELY(ompt_enabled.enabled)) {
1644  thread->th.ompt_thread_info = oldInfo;
1645  if (taskdata->td_flags.tiedness == TASK_TIED) {
1646  taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
1647  }
1648  __kmp_task_finish<true>(gtid, task, current_task);
1649  } else
1650 #endif
1651  __kmp_task_finish<false>(gtid, task, current_task);
1652  }
1653 
1654  KA_TRACE(
1655  30,
1656  ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n",
1657  gtid, taskdata, current_task));
1658  return;
1659 }
1660 
1661 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution
1662 //
1663 // loc_ref: location of original task pragma (ignored)
1664 // gtid: Global Thread ID of encountering thread
1665 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task''
1666 // Returns:
1667 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1668 // be resumed later.
1669 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1670 // resumed later.
1671 kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid,
1672  kmp_task_t *new_task) {
1673  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1674 
1675  KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid,
1676  loc_ref, new_taskdata));
1677 
1678 #if OMPT_SUPPORT
1679  kmp_taskdata_t *parent;
1680  if (UNLIKELY(ompt_enabled.enabled)) {
1681  parent = new_taskdata->td_parent;
1682  if (ompt_enabled.ompt_callback_task_create) {
1683  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1684  &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
1685  &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0,
1686  OMPT_GET_RETURN_ADDRESS(0));
1687  }
1688  }
1689 #endif
1690 
1691  /* Should we execute the new task or queue it? For now, let's just always try
1692  to queue it. If the queue fills up, then we'll execute it. */
1693 
1694  if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
1695  { // Execute this task immediately
1696  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1697  new_taskdata->td_flags.task_serial = 1;
1698  __kmp_invoke_task(gtid, new_task, current_task);
1699  }
1700 
1701  KA_TRACE(
1702  10,
1703  ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
1704  "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n",
1705  gtid, loc_ref, new_taskdata));
1706 
1707 #if OMPT_SUPPORT
1708  if (UNLIKELY(ompt_enabled.enabled)) {
1709  parent->ompt_task_info.frame.enter_frame = ompt_data_none;
1710  }
1711 #endif
1712  return TASK_CURRENT_NOT_QUEUED;
1713 }
1714 
1715 // __kmp_omp_task: Schedule a non-thread-switchable task for execution
1716 //
1717 // gtid: Global Thread ID of encountering thread
1718 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc()
1719 // serialize_immediate: if TRUE then if the task is executed immediately its
1720 // execution will be serialized
1721 // Returns:
1722 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1723 // be resumed later.
1724 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1725 // resumed later.
1726 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
1727  bool serialize_immediate) {
1728  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1729 
1730  /* Should we execute the new task or queue it? For now, let's just always try
1731  to queue it. If the queue fills up, then we'll execute it. */
1732  if (new_taskdata->td_flags.proxy == TASK_PROXY ||
1733  __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
1734  { // Execute this task immediately
1735  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1736  if (serialize_immediate)
1737  new_taskdata->td_flags.task_serial = 1;
1738  __kmp_invoke_task(gtid, new_task, current_task);
1739  }
1740 
1741  return TASK_CURRENT_NOT_QUEUED;
1742 }
1743 
1744 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a
1745 // non-thread-switchable task from the parent thread only!
1746 //
1747 // loc_ref: location of original task pragma (ignored)
1748 // gtid: Global Thread ID of encountering thread
1749 // new_task: non-thread-switchable task thunk allocated by
1750 // __kmp_omp_task_alloc()
1751 // Returns:
1752 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1753 // be resumed later.
1754 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1755 // resumed later.
1756 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
1757  kmp_task_t *new_task) {
1758  kmp_int32 res;
1759  KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK);
1760 
1761 #if KMP_DEBUG || OMPT_SUPPORT
1762  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1763 #endif
1764  KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
1765  new_taskdata));
1766  __kmp_assert_valid_gtid(gtid);
1767 
1768 #if OMPT_SUPPORT
1769  kmp_taskdata_t *parent = NULL;
1770  if (UNLIKELY(ompt_enabled.enabled)) {
1771  if (!new_taskdata->td_flags.started) {
1772  OMPT_STORE_RETURN_ADDRESS(gtid);
1773  parent = new_taskdata->td_parent;
1774  if (!parent->ompt_task_info.frame.enter_frame.ptr) {
1775  parent->ompt_task_info.frame.enter_frame.ptr =
1776  OMPT_GET_FRAME_ADDRESS(0);
1777  }
1778  if (ompt_enabled.ompt_callback_task_create) {
1779  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1780  &(parent->ompt_task_info.task_data),
1781  &(parent->ompt_task_info.frame),
1782  &(new_taskdata->ompt_task_info.task_data),
1783  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
1784  OMPT_LOAD_RETURN_ADDRESS(gtid));
1785  }
1786  } else {
1787  // We are scheduling the continuation of an UNTIED task.
1788  // Scheduling back to the parent task.
1789  __ompt_task_finish(new_task,
1790  new_taskdata->ompt_task_info.scheduling_parent,
1791  ompt_task_switch);
1792  new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
1793  }
1794  }
1795 #endif
1796 
1797  res = __kmp_omp_task(gtid, new_task, true);
1798 
1799  KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning "
1800  "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1801  gtid, loc_ref, new_taskdata));
1802 #if OMPT_SUPPORT
1803  if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
1804  parent->ompt_task_info.frame.enter_frame = ompt_data_none;
1805  }
1806 #endif
1807  return res;
1808 }
1809 
1810 // __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule
1811 // a taskloop task with the correct OMPT return address
1812 //
1813 // loc_ref: location of original task pragma (ignored)
1814 // gtid: Global Thread ID of encountering thread
1815 // new_task: non-thread-switchable task thunk allocated by
1816 // __kmp_omp_task_alloc()
1817 // codeptr_ra: return address for OMPT callback
1818 // Returns:
1819 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1820 // be resumed later.
1821 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1822 // resumed later.
1823 kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid,
1824  kmp_task_t *new_task, void *codeptr_ra) {
1825  kmp_int32 res;
1826  KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK);
1827 
1828 #if KMP_DEBUG || OMPT_SUPPORT
1829  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1830 #endif
1831  KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
1832  new_taskdata));
1833 
1834 #if OMPT_SUPPORT
1835  kmp_taskdata_t *parent = NULL;
1836  if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) {
1837  parent = new_taskdata->td_parent;
1838  if (!parent->ompt_task_info.frame.enter_frame.ptr)
1839  parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1840  if (ompt_enabled.ompt_callback_task_create) {
1841  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1842  &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
1843  &(new_taskdata->ompt_task_info.task_data),
1844  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
1845  codeptr_ra);
1846  }
1847  }
1848 #endif
1849 
1850  res = __kmp_omp_task(gtid, new_task, true);
1851 
1852  KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning "
1853  "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1854  gtid, loc_ref, new_taskdata));
1855 #if OMPT_SUPPORT
1856  if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
1857  parent->ompt_task_info.frame.enter_frame = ompt_data_none;
1858  }
1859 #endif
1860  return res;
1861 }
1862 
1863 template <bool ompt>
1864 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
1865  void *frame_address,
1866  void *return_address) {
1867  kmp_taskdata_t *taskdata = nullptr;
1868  kmp_info_t *thread;
1869  int thread_finished = FALSE;
1870  KMP_SET_THREAD_STATE_BLOCK(TASKWAIT);
1871 
1872  KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref));
1873  KMP_DEBUG_ASSERT(gtid >= 0);
1874 
1875  if (__kmp_tasking_mode != tskm_immediate_exec) {
1876  thread = __kmp_threads[gtid];
1877  taskdata = thread->th.th_current_task;
1878 
1879 #if OMPT_SUPPORT && OMPT_OPTIONAL
1880  ompt_data_t *my_task_data;
1881  ompt_data_t *my_parallel_data;
1882 
1883  if (ompt) {
1884  my_task_data = &(taskdata->ompt_task_info.task_data);
1885  my_parallel_data = OMPT_CUR_TEAM_DATA(thread);
1886 
1887  taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address;
1888 
1889  if (ompt_enabled.ompt_callback_sync_region) {
1890  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
1891  ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
1892  my_task_data, return_address);
1893  }
1894 
1895  if (ompt_enabled.ompt_callback_sync_region_wait) {
1896  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
1897  ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
1898  my_task_data, return_address);
1899  }
1900  }
1901 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
1902 
1903 // Debugger: The taskwait is active. Store location and thread encountered the
1904 // taskwait.
1905 #if USE_ITT_BUILD
1906 // Note: These values are used by ITT events as well.
1907 #endif /* USE_ITT_BUILD */
1908  taskdata->td_taskwait_counter += 1;
1909  taskdata->td_taskwait_ident = loc_ref;
1910  taskdata->td_taskwait_thread = gtid + 1;
1911 
1912 #if USE_ITT_BUILD
1913  void *itt_sync_obj = NULL;
1914 #if USE_ITT_NOTIFY
1915  KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
1916 #endif /* USE_ITT_NOTIFY */
1917 #endif /* USE_ITT_BUILD */
1918 
1919  bool must_wait =
1920  !taskdata->td_flags.team_serial && !taskdata->td_flags.final;
1921 
1922  must_wait = must_wait || (thread->th.th_task_team != NULL &&
1923  thread->th.th_task_team->tt.tt_found_proxy_tasks);
1924  // If hidden helper thread is encountered, we must enable wait here.
1925  must_wait =
1926  must_wait ||
1927  (__kmp_enable_hidden_helper && thread->th.th_task_team != NULL &&
1928  thread->th.th_task_team->tt.tt_hidden_helper_task_encountered);
1929 
1930  if (must_wait) {
1931  kmp_flag_32<false, false> flag(
1932  RCAST(std::atomic<kmp_uint32> *,
1933  &(taskdata->td_incomplete_child_tasks)),
1934  0U);
1935  while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) {
1936  flag.execute_tasks(thread, gtid, FALSE,
1937  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
1938  __kmp_task_stealing_constraint);
1939  }
1940  }
1941 #if USE_ITT_BUILD
1942  KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
1943  KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with children
1944 #endif /* USE_ITT_BUILD */
1945 
1946  // Debugger: The taskwait is completed. Location remains, but thread is
1947  // negated.
1948  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
1949 
1950 #if OMPT_SUPPORT && OMPT_OPTIONAL
1951  if (ompt) {
1952  if (ompt_enabled.ompt_callback_sync_region_wait) {
1953  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
1954  ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
1955  my_task_data, return_address);
1956  }
1957  if (ompt_enabled.ompt_callback_sync_region) {
1958  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
1959  ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
1960  my_task_data, return_address);
1961  }
1962  taskdata->ompt_task_info.frame.enter_frame = ompt_data_none;
1963  }
1964 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
1965 
1966  }
1967 
1968  KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
1969  "returning TASK_CURRENT_NOT_QUEUED\n",
1970  gtid, taskdata));
1971 
1972  return TASK_CURRENT_NOT_QUEUED;
1973 }
1974 
1975 #if OMPT_SUPPORT && OMPT_OPTIONAL
1976 OMPT_NOINLINE
1977 static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid,
1978  void *frame_address,
1979  void *return_address) {
1980  return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address,
1981  return_address);
1982 }
1983 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
1984 
1985 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are
1986 // complete
1987 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) {
1988 #if OMPT_SUPPORT && OMPT_OPTIONAL
1989  if (UNLIKELY(ompt_enabled.enabled)) {
1990  OMPT_STORE_RETURN_ADDRESS(gtid);
1991  return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0),
1992  OMPT_LOAD_RETURN_ADDRESS(gtid));
1993  }
1994 #endif
1995  return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL);
1996 }
1997 
1998 // __kmpc_omp_taskyield: switch to a different task
1999 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
2000  kmp_taskdata_t *taskdata = NULL;
2001  kmp_info_t *thread;
2002  int thread_finished = FALSE;
2003 
2004  KMP_COUNT_BLOCK(OMP_TASKYIELD);
2005  KMP_SET_THREAD_STATE_BLOCK(TASKYIELD);
2006 
2007  KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
2008  gtid, loc_ref, end_part));
2009  __kmp_assert_valid_gtid(gtid);
2010 
2011  if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) {
2012  thread = __kmp_threads[gtid];
2013  taskdata = thread->th.th_current_task;
2014 // Should we model this as a task wait or not?
2015 // Debugger: The taskwait is active. Store location and thread encountered the
2016 // taskwait.
2017 #if USE_ITT_BUILD
2018 // Note: These values are used by ITT events as well.
2019 #endif /* USE_ITT_BUILD */
2020  taskdata->td_taskwait_counter += 1;
2021  taskdata->td_taskwait_ident = loc_ref;
2022  taskdata->td_taskwait_thread = gtid + 1;
2023 
2024 #if USE_ITT_BUILD
2025  void *itt_sync_obj = NULL;
2026 #if USE_ITT_NOTIFY
2027  KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2028 #endif /* USE_ITT_NOTIFY */
2029 #endif /* USE_ITT_BUILD */
2030  if (!taskdata->td_flags.team_serial) {
2031  kmp_task_team_t *task_team = thread->th.th_task_team;
2032  if (task_team != NULL) {
2033  if (KMP_TASKING_ENABLED(task_team)) {
2034 #if OMPT_SUPPORT
2035  if (UNLIKELY(ompt_enabled.enabled))
2036  thread->th.ompt_thread_info.ompt_task_yielded = 1;
2037 #endif
2038  __kmp_execute_tasks_32(
2039  thread, gtid, (kmp_flag_32<> *)NULL, FALSE,
2040  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2041  __kmp_task_stealing_constraint);
2042 #if OMPT_SUPPORT
2043  if (UNLIKELY(ompt_enabled.enabled))
2044  thread->th.ompt_thread_info.ompt_task_yielded = 0;
2045 #endif
2046  }
2047  }
2048  }
2049 #if USE_ITT_BUILD
2050  KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2051 #endif /* USE_ITT_BUILD */
2052 
2053  // Debugger: The taskwait is completed. Location remains, but thread is
2054  // negated.
2055  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
2056  }
2057 
2058  KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
2059  "returning TASK_CURRENT_NOT_QUEUED\n",
2060  gtid, taskdata));
2061 
2062  return TASK_CURRENT_NOT_QUEUED;
2063 }
2064 
2065 // Task Reduction implementation
2066 //
2067 // Note: initial implementation didn't take into account the possibility
2068 // to specify omp_orig for initializer of the UDR (user defined reduction).
2069 // Corrected implementation takes into account the omp_orig object.
2070 // Compiler is free to use old implementation if omp_orig is not specified.
2071 
2080 typedef struct kmp_taskred_flags {
2082  unsigned lazy_priv : 1;
2083  unsigned reserved31 : 31;
2085 
2089 typedef struct kmp_task_red_input {
2090  void *reduce_shar;
2091  size_t reduce_size;
2092  // three compiler-generated routines (init, fini are optional):
2093  void *reduce_init;
2094  void *reduce_fini;
2095  void *reduce_comb;
2098 
2102 typedef struct kmp_taskred_data {
2103  void *reduce_shar;
2104  size_t reduce_size;
2106  void *reduce_priv;
2107  void *reduce_pend;
2108  // three compiler-generated routines (init, fini are optional):
2109  void *reduce_comb;
2110  void *reduce_init;
2111  void *reduce_fini;
2112  void *reduce_orig;
2114 
2120 typedef struct kmp_taskred_input {
2121  void *reduce_shar;
2122  void *reduce_orig;
2123  size_t reduce_size;
2124  // three compiler-generated routines (init, fini are optional):
2125  void *reduce_init;
2126  void *reduce_fini;
2127  void *reduce_comb;
2134 template <typename T> void __kmp_assign_orig(kmp_taskred_data_t &item, T &src);
2135 template <>
2136 void __kmp_assign_orig<kmp_task_red_input_t>(kmp_taskred_data_t &item,
2137  kmp_task_red_input_t &src) {
2138  item.reduce_orig = NULL;
2139 }
2140 template <>
2141 void __kmp_assign_orig<kmp_taskred_input_t>(kmp_taskred_data_t &item,
2142  kmp_taskred_input_t &src) {
2143  if (src.reduce_orig != NULL) {
2144  item.reduce_orig = src.reduce_orig;
2145  } else {
2146  item.reduce_orig = src.reduce_shar;
2147  } // non-NULL reduce_orig means new interface used
2148 }
2149 
2150 template <typename T> void __kmp_call_init(kmp_taskred_data_t &item, size_t j);
2151 template <>
2152 void __kmp_call_init<kmp_task_red_input_t>(kmp_taskred_data_t &item,
2153  size_t offset) {
2154  ((void (*)(void *))item.reduce_init)((char *)(item.reduce_priv) + offset);
2155 }
2156 template <>
2157 void __kmp_call_init<kmp_taskred_input_t>(kmp_taskred_data_t &item,
2158  size_t offset) {
2159  ((void (*)(void *, void *))item.reduce_init)(
2160  (char *)(item.reduce_priv) + offset, item.reduce_orig);
2161 }
2162 
2163 template <typename T>
2164 void *__kmp_task_reduction_init(int gtid, int num, T *data) {
2165  __kmp_assert_valid_gtid(gtid);
2166  kmp_info_t *thread = __kmp_threads[gtid];
2167  kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup;
2168  kmp_uint32 nth = thread->th.th_team_nproc;
2169  kmp_taskred_data_t *arr;
2170 
2171  // check input data just in case
2172  KMP_ASSERT(tg != NULL);
2173  KMP_ASSERT(data != NULL);
2174  KMP_ASSERT(num > 0);
2175  if (nth == 1) {
2176  KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n",
2177  gtid, tg));
2178  return (void *)tg;
2179  }
2180  KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n",
2181  gtid, tg, num));
2182  arr = (kmp_taskred_data_t *)__kmp_thread_malloc(
2183  thread, num * sizeof(kmp_taskred_data_t));
2184  for (int i = 0; i < num; ++i) {
2185  size_t size = data[i].reduce_size - 1;
2186  // round the size up to cache line per thread-specific item
2187  size += CACHE_LINE - size % CACHE_LINE;
2188  KMP_ASSERT(data[i].reduce_comb != NULL); // combiner is mandatory
2189  arr[i].reduce_shar = data[i].reduce_shar;
2190  arr[i].reduce_size = size;
2191  arr[i].flags = data[i].flags;
2192  arr[i].reduce_comb = data[i].reduce_comb;
2193  arr[i].reduce_init = data[i].reduce_init;
2194  arr[i].reduce_fini = data[i].reduce_fini;
2195  __kmp_assign_orig<T>(arr[i], data[i]);
2196  if (!arr[i].flags.lazy_priv) {
2197  // allocate cache-line aligned block and fill it with zeros
2198  arr[i].reduce_priv = __kmp_allocate(nth * size);
2199  arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size;
2200  if (arr[i].reduce_init != NULL) {
2201  // initialize all thread-specific items
2202  for (size_t j = 0; j < nth; ++j) {
2203  __kmp_call_init<T>(arr[i], j * size);
2204  }
2205  }
2206  } else {
2207  // only allocate space for pointers now,
2208  // objects will be lazily allocated/initialized if/when requested
2209  // note that __kmp_allocate zeroes the allocated memory
2210  arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *));
2211  }
2212  }
2213  tg->reduce_data = (void *)arr;
2214  tg->reduce_num_data = num;
2215  return (void *)tg;
2216 }
2217 
2232 void *__kmpc_task_reduction_init(int gtid, int num, void *data) {
2233  return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data);
2234 }
2235 
2248 void *__kmpc_taskred_init(int gtid, int num, void *data) {
2249  return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
2250 }
2251 
2252 // Copy task reduction data (except for shared pointers).
2253 template <typename T>
2254 void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data,
2255  kmp_taskgroup_t *tg, void *reduce_data) {
2256  kmp_taskred_data_t *arr;
2257  KA_TRACE(20, ("__kmp_task_reduction_init_copy: Th %p, init taskgroup %p,"
2258  " from data %p\n",
2259  thr, tg, reduce_data));
2260  arr = (kmp_taskred_data_t *)__kmp_thread_malloc(
2261  thr, num * sizeof(kmp_taskred_data_t));
2262  // threads will share private copies, thunk routines, sizes, flags, etc.:
2263  KMP_MEMCPY(arr, reduce_data, num * sizeof(kmp_taskred_data_t));
2264  for (int i = 0; i < num; ++i) {
2265  arr[i].reduce_shar = data[i].reduce_shar; // init unique shared pointers
2266  }
2267  tg->reduce_data = (void *)arr;
2268  tg->reduce_num_data = num;
2269 }
2270 
2280 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
2281  __kmp_assert_valid_gtid(gtid);
2282  kmp_info_t *thread = __kmp_threads[gtid];
2283  kmp_int32 nth = thread->th.th_team_nproc;
2284  if (nth == 1)
2285  return data; // nothing to do
2286 
2287  kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp;
2288  if (tg == NULL)
2289  tg = thread->th.th_current_task->td_taskgroup;
2290  KMP_ASSERT(tg != NULL);
2291  kmp_taskred_data_t *arr = (kmp_taskred_data_t *)(tg->reduce_data);
2292  kmp_int32 num = tg->reduce_num_data;
2293  kmp_int32 tid = thread->th.th_info.ds.ds_tid;
2294 
2295  KMP_ASSERT(data != NULL);
2296  while (tg != NULL) {
2297  for (int i = 0; i < num; ++i) {
2298  if (!arr[i].flags.lazy_priv) {
2299  if (data == arr[i].reduce_shar ||
2300  (data >= arr[i].reduce_priv && data < arr[i].reduce_pend))
2301  return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size;
2302  } else {
2303  // check shared location first
2304  void **p_priv = (void **)(arr[i].reduce_priv);
2305  if (data == arr[i].reduce_shar)
2306  goto found;
2307  // check if we get some thread specific location as parameter
2308  for (int j = 0; j < nth; ++j)
2309  if (data == p_priv[j])
2310  goto found;
2311  continue; // not found, continue search
2312  found:
2313  if (p_priv[tid] == NULL) {
2314  // allocate thread specific object lazily
2315  p_priv[tid] = __kmp_allocate(arr[i].reduce_size);
2316  if (arr[i].reduce_init != NULL) {
2317  if (arr[i].reduce_orig != NULL) { // new interface
2318  ((void (*)(void *, void *))arr[i].reduce_init)(
2319  p_priv[tid], arr[i].reduce_orig);
2320  } else { // old interface (single parameter)
2321  ((void (*)(void *))arr[i].reduce_init)(p_priv[tid]);
2322  }
2323  }
2324  }
2325  return p_priv[tid];
2326  }
2327  }
2328  tg = tg->parent;
2329  arr = (kmp_taskred_data_t *)(tg->reduce_data);
2330  num = tg->reduce_num_data;
2331  }
2332  KMP_ASSERT2(0, "Unknown task reduction item");
2333  return NULL; // ERROR, this line never executed
2334 }
2335 
2336 // Finalize task reduction.
2337 // Called from __kmpc_end_taskgroup()
2338 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) {
2339  kmp_int32 nth = th->th.th_team_nproc;
2340  KMP_DEBUG_ASSERT(nth > 1); // should not be called if nth == 1
2341  kmp_taskred_data_t *arr = (kmp_taskred_data_t *)tg->reduce_data;
2342  kmp_int32 num = tg->reduce_num_data;
2343  for (int i = 0; i < num; ++i) {
2344  void *sh_data = arr[i].reduce_shar;
2345  void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini);
2346  void (*f_comb)(void *, void *) =
2347  (void (*)(void *, void *))(arr[i].reduce_comb);
2348  if (!arr[i].flags.lazy_priv) {
2349  void *pr_data = arr[i].reduce_priv;
2350  size_t size = arr[i].reduce_size;
2351  for (int j = 0; j < nth; ++j) {
2352  void *priv_data = (char *)pr_data + j * size;
2353  f_comb(sh_data, priv_data); // combine results
2354  if (f_fini)
2355  f_fini(priv_data); // finalize if needed
2356  }
2357  } else {
2358  void **pr_data = (void **)(arr[i].reduce_priv);
2359  for (int j = 0; j < nth; ++j) {
2360  if (pr_data[j] != NULL) {
2361  f_comb(sh_data, pr_data[j]); // combine results
2362  if (f_fini)
2363  f_fini(pr_data[j]); // finalize if needed
2364  __kmp_free(pr_data[j]);
2365  }
2366  }
2367  }
2368  __kmp_free(arr[i].reduce_priv);
2369  }
2370  __kmp_thread_free(th, arr);
2371  tg->reduce_data = NULL;
2372  tg->reduce_num_data = 0;
2373 }
2374 
2375 // Cleanup task reduction data for parallel or worksharing,
2376 // do not touch task private data other threads still working with.
2377 // Called from __kmpc_end_taskgroup()
2378 static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) {
2379  __kmp_thread_free(th, tg->reduce_data);
2380  tg->reduce_data = NULL;
2381  tg->reduce_num_data = 0;
2382 }
2383 
2384 template <typename T>
2385 void *__kmp_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws,
2386  int num, T *data) {
2387  __kmp_assert_valid_gtid(gtid);
2388  kmp_info_t *thr = __kmp_threads[gtid];
2389  kmp_int32 nth = thr->th.th_team_nproc;
2390  __kmpc_taskgroup(loc, gtid); // form new taskgroup first
2391  if (nth == 1) {
2392  KA_TRACE(10,
2393  ("__kmpc_reduction_modifier_init: T#%d, tg %p, exiting nth=1\n",
2394  gtid, thr->th.th_current_task->td_taskgroup));
2395  return (void *)thr->th.th_current_task->td_taskgroup;
2396  }
2397  kmp_team_t *team = thr->th.th_team;
2398  void *reduce_data;
2399  kmp_taskgroup_t *tg;
2400  reduce_data = KMP_ATOMIC_LD_RLX(&team->t.t_tg_reduce_data[is_ws]);
2401  if (reduce_data == NULL &&
2402  __kmp_atomic_compare_store(&team->t.t_tg_reduce_data[is_ws], reduce_data,
2403  (void *)1)) {
2404  // single thread enters this block to initialize common reduction data
2405  KMP_DEBUG_ASSERT(reduce_data == NULL);
2406  // first initialize own data, then make a copy other threads can use
2407  tg = (kmp_taskgroup_t *)__kmp_task_reduction_init<T>(gtid, num, data);
2408  reduce_data = __kmp_thread_malloc(thr, num * sizeof(kmp_taskred_data_t));
2409  KMP_MEMCPY(reduce_data, tg->reduce_data, num * sizeof(kmp_taskred_data_t));
2410  // fini counters should be 0 at this point
2411  KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[0]) == 0);
2412  KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[1]) == 0);
2413  KMP_ATOMIC_ST_REL(&team->t.t_tg_reduce_data[is_ws], reduce_data);
2414  } else {
2415  while (
2416  (reduce_data = KMP_ATOMIC_LD_ACQ(&team->t.t_tg_reduce_data[is_ws])) ==
2417  (void *)1) { // wait for task reduction initialization
2418  KMP_CPU_PAUSE();
2419  }
2420  KMP_DEBUG_ASSERT(reduce_data > (void *)1); // should be valid pointer here
2421  tg = thr->th.th_current_task->td_taskgroup;
2422  __kmp_task_reduction_init_copy<T>(thr, num, data, tg, reduce_data);
2423  }
2424  return tg;
2425 }
2426 
2443 void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws,
2444  int num, void *data) {
2445  return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num,
2446  (kmp_task_red_input_t *)data);
2447 }
2448 
2463 void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num,
2464  void *data) {
2465  return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num,
2466  (kmp_taskred_input_t *)data);
2467 }
2468 
2477 void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws) {
2478  __kmpc_end_taskgroup(loc, gtid);
2479 }
2480 
2481 // __kmpc_taskgroup: Start a new taskgroup
2482 void __kmpc_taskgroup(ident_t *loc, int gtid) {
2483  __kmp_assert_valid_gtid(gtid);
2484  kmp_info_t *thread = __kmp_threads[gtid];
2485  kmp_taskdata_t *taskdata = thread->th.th_current_task;
2486  kmp_taskgroup_t *tg_new =
2487  (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t));
2488  KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new));
2489  KMP_ATOMIC_ST_RLX(&tg_new->count, 0);
2490  KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq);
2491  tg_new->parent = taskdata->td_taskgroup;
2492  tg_new->reduce_data = NULL;
2493  tg_new->reduce_num_data = 0;
2494  tg_new->gomp_data = NULL;
2495  taskdata->td_taskgroup = tg_new;
2496 
2497 #if OMPT_SUPPORT && OMPT_OPTIONAL
2498  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
2499  void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2500  if (!codeptr)
2501  codeptr = OMPT_GET_RETURN_ADDRESS(0);
2502  kmp_team_t *team = thread->th.th_team;
2503  ompt_data_t my_task_data = taskdata->ompt_task_info.task_data;
2504  // FIXME: I think this is wrong for lwt!
2505  ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data;
2506 
2507  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2508  ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2509  &(my_task_data), codeptr);
2510  }
2511 #endif
2512 }
2513 
2514 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task
2515 // and its descendants are complete
2516 void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
2517  __kmp_assert_valid_gtid(gtid);
2518  kmp_info_t *thread = __kmp_threads[gtid];
2519  kmp_taskdata_t *taskdata = thread->th.th_current_task;
2520  kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
2521  int thread_finished = FALSE;
2522 
2523 #if OMPT_SUPPORT && OMPT_OPTIONAL
2524  kmp_team_t *team;
2525  ompt_data_t my_task_data;
2526  ompt_data_t my_parallel_data;
2527  void *codeptr = nullptr;
2528  if (UNLIKELY(ompt_enabled.enabled)) {
2529  team = thread->th.th_team;
2530  my_task_data = taskdata->ompt_task_info.task_data;
2531  // FIXME: I think this is wrong for lwt!
2532  my_parallel_data = team->t.ompt_team_info.parallel_data;
2533  codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2534  if (!codeptr)
2535  codeptr = OMPT_GET_RETURN_ADDRESS(0);
2536  }
2537 #endif
2538 
2539  KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc));
2540  KMP_DEBUG_ASSERT(taskgroup != NULL);
2541  KMP_SET_THREAD_STATE_BLOCK(TASKGROUP);
2542 
2543  if (__kmp_tasking_mode != tskm_immediate_exec) {
2544  // mark task as waiting not on a barrier
2545  taskdata->td_taskwait_counter += 1;
2546  taskdata->td_taskwait_ident = loc;
2547  taskdata->td_taskwait_thread = gtid + 1;
2548 #if USE_ITT_BUILD
2549  // For ITT the taskgroup wait is similar to taskwait until we need to
2550  // distinguish them
2551  void *itt_sync_obj = NULL;
2552 #if USE_ITT_NOTIFY
2553  KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2554 #endif /* USE_ITT_NOTIFY */
2555 #endif /* USE_ITT_BUILD */
2556 
2557 #if OMPT_SUPPORT && OMPT_OPTIONAL
2558  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2559  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2560  ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2561  &(my_task_data), codeptr);
2562  }
2563 #endif
2564 
2565  if (!taskdata->td_flags.team_serial ||
2566  (thread->th.th_task_team != NULL &&
2567  (thread->th.th_task_team->tt.tt_found_proxy_tasks ||
2568  thread->th.th_task_team->tt.tt_hidden_helper_task_encountered))) {
2569  kmp_flag_32<false, false> flag(
2570  RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 0U);
2571  while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) {
2572  flag.execute_tasks(thread, gtid, FALSE,
2573  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2574  __kmp_task_stealing_constraint);
2575  }
2576  }
2577  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting
2578 
2579 #if OMPT_SUPPORT && OMPT_OPTIONAL
2580  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2581  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2582  ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2583  &(my_task_data), codeptr);
2584  }
2585 #endif
2586 
2587 #if USE_ITT_BUILD
2588  KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2589  KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with descendants
2590 #endif /* USE_ITT_BUILD */
2591  }
2592  KMP_DEBUG_ASSERT(taskgroup->count == 0);
2593 
2594  if (taskgroup->reduce_data != NULL &&
2595  !taskgroup->gomp_data) { // need to reduce?
2596  int cnt;
2597  void *reduce_data;
2598  kmp_team_t *t = thread->th.th_team;
2599  kmp_taskred_data_t *arr = (kmp_taskred_data_t *)taskgroup->reduce_data;
2600  // check if <priv> data of the first reduction variable shared for the team
2601  void *priv0 = arr[0].reduce_priv;
2602  if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[0])) != NULL &&
2603  ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) {
2604  // finishing task reduction on parallel
2605  cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[0]);
2606  if (cnt == thread->th.th_team_nproc - 1) {
2607  // we are the last thread passing __kmpc_reduction_modifier_fini()
2608  // finalize task reduction:
2609  __kmp_task_reduction_fini(thread, taskgroup);
2610  // cleanup fields in the team structure:
2611  // TODO: is relaxed store enough here (whole barrier should follow)?
2612  __kmp_thread_free(thread, reduce_data);
2613  KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[0], NULL);
2614  KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[0], 0);
2615  } else {
2616  // we are not the last thread passing __kmpc_reduction_modifier_fini(),
2617  // so do not finalize reduction, just clean own copy of the data
2618  __kmp_task_reduction_clean(thread, taskgroup);
2619  }
2620  } else if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[1])) !=
2621  NULL &&
2622  ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) {
2623  // finishing task reduction on worksharing
2624  cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[1]);
2625  if (cnt == thread->th.th_team_nproc - 1) {
2626  // we are the last thread passing __kmpc_reduction_modifier_fini()
2627  __kmp_task_reduction_fini(thread, taskgroup);
2628  // cleanup fields in team structure:
2629  // TODO: is relaxed store enough here (whole barrier should follow)?
2630  __kmp_thread_free(thread, reduce_data);
2631  KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[1], NULL);
2632  KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[1], 0);
2633  } else {
2634  // we are not the last thread passing __kmpc_reduction_modifier_fini(),
2635  // so do not finalize reduction, just clean own copy of the data
2636  __kmp_task_reduction_clean(thread, taskgroup);
2637  }
2638  } else {
2639  // finishing task reduction on taskgroup
2640  __kmp_task_reduction_fini(thread, taskgroup);
2641  }
2642  }
2643  // Restore parent taskgroup for the current task
2644  taskdata->td_taskgroup = taskgroup->parent;
2645  __kmp_thread_free(thread, taskgroup);
2646 
2647  KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n",
2648  gtid, taskdata));
2649 
2650 #if OMPT_SUPPORT && OMPT_OPTIONAL
2651  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
2652  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2653  ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2654  &(my_task_data), codeptr);
2655  }
2656 #endif
2657 }
2658 
2659 // __kmp_remove_my_task: remove a task from my own deque
2660 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid,
2661  kmp_task_team_t *task_team,
2662  kmp_int32 is_constrained) {
2663  kmp_task_t *task;
2664  kmp_taskdata_t *taskdata;
2665  kmp_thread_data_t *thread_data;
2666  kmp_uint32 tail;
2667 
2668  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2669  KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data !=
2670  NULL); // Caller should check this condition
2671 
2672  thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)];
2673 
2674  KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",
2675  gtid, thread_data->td.td_deque_ntasks,
2676  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2677 
2678  if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
2679  KA_TRACE(10,
2680  ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
2681  "ntasks=%d head=%u tail=%u\n",
2682  gtid, thread_data->td.td_deque_ntasks,
2683  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2684  return NULL;
2685  }
2686 
2687  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2688 
2689  if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
2690  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2691  KA_TRACE(10,
2692  ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
2693  "ntasks=%d head=%u tail=%u\n",
2694  gtid, thread_data->td.td_deque_ntasks,
2695  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2696  return NULL;
2697  }
2698 
2699  tail = (thread_data->td.td_deque_tail - 1) &
2700  TASK_DEQUE_MASK(thread_data->td); // Wrap index.
2701  taskdata = thread_data->td.td_deque[tail];
2702 
2703  if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata,
2704  thread->th.th_current_task)) {
2705  // The TSC does not allow to steal victim task
2706  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2707  KA_TRACE(10,
2708  ("__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: "
2709  "ntasks=%d head=%u tail=%u\n",
2710  gtid, thread_data->td.td_deque_ntasks,
2711  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2712  return NULL;
2713  }
2714 
2715  thread_data->td.td_deque_tail = tail;
2716  TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1);
2717 
2718  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2719 
2720  KA_TRACE(10, ("__kmp_remove_my_task(exit #4): T#%d task %p removed: "
2721  "ntasks=%d head=%u tail=%u\n",
2722  gtid, taskdata, thread_data->td.td_deque_ntasks,
2723  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2724 
2725  task = KMP_TASKDATA_TO_TASK(taskdata);
2726  return task;
2727 }
2728 
2729 // __kmp_steal_task: remove a task from another thread's deque
2730 // Assume that calling thread has already checked existence of
2731 // task_team thread_data before calling this routine.
2732 static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid,
2733  kmp_task_team_t *task_team,
2734  std::atomic<kmp_int32> *unfinished_threads,
2735  int *thread_finished,
2736  kmp_int32 is_constrained) {
2737  kmp_task_t *task;
2738  kmp_taskdata_t *taskdata;
2739  kmp_taskdata_t *current;
2740  kmp_thread_data_t *victim_td, *threads_data;
2741  kmp_int32 target;
2742  kmp_int32 victim_tid;
2743 
2744  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2745 
2746  threads_data = task_team->tt.tt_threads_data;
2747  KMP_DEBUG_ASSERT(threads_data != NULL); // Caller should check this condition
2748 
2749  victim_tid = victim_thr->th.th_info.ds.ds_tid;
2750  victim_td = &threads_data[victim_tid];
2751 
2752  KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
2753  "task_team=%p ntasks=%d head=%u tail=%u\n",
2754  gtid, __kmp_gtid_from_thread(victim_thr), task_team,
2755  victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
2756  victim_td->td.td_deque_tail));
2757 
2758  if (TCR_4(victim_td->td.td_deque_ntasks) == 0) {
2759  KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
2760  "task_team=%p ntasks=%d head=%u tail=%u\n",
2761  gtid, __kmp_gtid_from_thread(victim_thr), task_team,
2762  victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
2763  victim_td->td.td_deque_tail));
2764  return NULL;
2765  }
2766 
2767  __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock);
2768 
2769  int ntasks = TCR_4(victim_td->td.td_deque_ntasks);
2770  // Check again after we acquire the lock
2771  if (ntasks == 0) {
2772  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2773  KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
2774  "task_team=%p ntasks=%d head=%u tail=%u\n",
2775  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
2776  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2777  return NULL;
2778  }
2779 
2780  KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL);
2781  current = __kmp_threads[gtid]->th.th_current_task;
2782  taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head];
2783  if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
2784  // Bump head pointer and Wrap.
2785  victim_td->td.td_deque_head =
2786  (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td);
2787  } else {
2788  if (!task_team->tt.tt_untied_task_encountered) {
2789  // The TSC does not allow to steal victim task
2790  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2791  KA_TRACE(10, ("__kmp_steal_task(exit #3): T#%d could not steal from "
2792  "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
2793  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
2794  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2795  return NULL;
2796  }
2797  int i;
2798  // walk through victim's deque trying to steal any task
2799  target = victim_td->td.td_deque_head;
2800  taskdata = NULL;
2801  for (i = 1; i < ntasks; ++i) {
2802  target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
2803  taskdata = victim_td->td.td_deque[target];
2804  if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
2805  break; // found victim task
2806  } else {
2807  taskdata = NULL;
2808  }
2809  }
2810  if (taskdata == NULL) {
2811  // No appropriate candidate to steal found
2812  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2813  KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from "
2814  "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
2815  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
2816  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2817  return NULL;
2818  }
2819  int prev = target;
2820  for (i = i + 1; i < ntasks; ++i) {
2821  // shift remaining tasks in the deque left by 1
2822  target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
2823  victim_td->td.td_deque[prev] = victim_td->td.td_deque[target];
2824  prev = target;
2825  }
2826  KMP_DEBUG_ASSERT(
2827  victim_td->td.td_deque_tail ==
2828  (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td)));
2829  victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped))
2830  }
2831  if (*thread_finished) {
2832  // We need to un-mark this victim as a finished victim. This must be done
2833  // before releasing the lock, or else other threads (starting with the
2834  // primary thread victim) might be prematurely released from the barrier!!!
2835  kmp_int32 count;
2836 
2837  count = KMP_ATOMIC_INC(unfinished_threads);
2838 
2839  KA_TRACE(
2840  20,
2841  ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
2842  gtid, count + 1, task_team));
2843 
2844  *thread_finished = FALSE;
2845  }
2846  TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1);
2847 
2848  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2849 
2850  KMP_COUNT_BLOCK(TASK_stolen);
2851  KA_TRACE(10,
2852  ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
2853  "task_team=%p ntasks=%d head=%u tail=%u\n",
2854  gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team,
2855  ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2856 
2857  task = KMP_TASKDATA_TO_TASK(taskdata);
2858  return task;
2859 }
2860 
2861 // __kmp_execute_tasks_template: Choose and execute tasks until either the
2862 // condition is statisfied (return true) or there are none left (return false).
2863 //
2864 // final_spin is TRUE if this is the spin at the release barrier.
2865 // thread_finished indicates whether the thread is finished executing all
2866 // the tasks it has on its deque, and is at the release barrier.
2867 // spinner is the location on which to spin.
2868 // spinner == NULL means only execute a single task and return.
2869 // checker is the value to check to terminate the spin.
2870 template <class C>
2871 static inline int __kmp_execute_tasks_template(
2872  kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin,
2873  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
2874  kmp_int32 is_constrained) {
2875  kmp_task_team_t *task_team = thread->th.th_task_team;
2876  kmp_thread_data_t *threads_data;
2877  kmp_task_t *task;
2878  kmp_info_t *other_thread;
2879  kmp_taskdata_t *current_task = thread->th.th_current_task;
2880  std::atomic<kmp_int32> *unfinished_threads;
2881  kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0,
2882  tid = thread->th.th_info.ds.ds_tid;
2883 
2884  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2885  KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]);
2886 
2887  if (task_team == NULL || current_task == NULL)
2888  return FALSE;
2889 
2890  KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d "
2891  "*thread_finished=%d\n",
2892  gtid, final_spin, *thread_finished));
2893 
2894  thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
2895  threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
2896 
2897  KMP_DEBUG_ASSERT(threads_data != NULL);
2898 
2899  nthreads = task_team->tt.tt_nproc;
2900  unfinished_threads = &(task_team->tt.tt_unfinished_threads);
2901  KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks ||
2902  task_team->tt.tt_hidden_helper_task_encountered);
2903  KMP_DEBUG_ASSERT(*unfinished_threads >= 0);
2904 
2905  while (1) { // Outer loop keeps trying to find tasks in case of single thread
2906  // getting tasks from target constructs
2907  while (1) { // Inner loop to find a task and execute it
2908  task = NULL;
2909  if (use_own_tasks) { // check on own queue first
2910  task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained);
2911  }
2912  if ((task == NULL) && (nthreads > 1)) { // Steal a task
2913  int asleep = 1;
2914  use_own_tasks = 0;
2915  // Try to steal from the last place I stole from successfully.
2916  if (victim_tid == -2) { // haven't stolen anything yet
2917  victim_tid = threads_data[tid].td.td_deque_last_stolen;
2918  if (victim_tid !=
2919  -1) // if we have a last stolen from victim, get the thread
2920  other_thread = threads_data[victim_tid].td.td_thr;
2921  }
2922  if (victim_tid != -1) { // found last victim
2923  asleep = 0;
2924  } else if (!new_victim) { // no recent steals and we haven't already
2925  // used a new victim; select a random thread
2926  do { // Find a different thread to steal work from.
2927  // Pick a random thread. Initial plan was to cycle through all the
2928  // threads, and only return if we tried to steal from every thread,
2929  // and failed. Arch says that's not such a great idea.
2930  victim_tid = __kmp_get_random(thread) % (nthreads - 1);
2931  if (victim_tid >= tid) {
2932  ++victim_tid; // Adjusts random distribution to exclude self
2933  }
2934  // Found a potential victim
2935  other_thread = threads_data[victim_tid].td.td_thr;
2936  // There is a slight chance that __kmp_enable_tasking() did not wake
2937  // up all threads waiting at the barrier. If victim is sleeping,
2938  // then wake it up. Since we were going to pay the cache miss
2939  // penalty for referencing another thread's kmp_info_t struct
2940  // anyway,
2941  // the check shouldn't cost too much performance at this point. In
2942  // extra barrier mode, tasks do not sleep at the separate tasking
2943  // barrier, so this isn't a problem.
2944  asleep = 0;
2945  if ((__kmp_tasking_mode == tskm_task_teams) &&
2946  (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) &&
2947  (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc)) !=
2948  NULL)) {
2949  asleep = 1;
2950  __kmp_null_resume_wrapper(__kmp_gtid_from_thread(other_thread),
2951  other_thread->th.th_sleep_loc);
2952  // A sleeping thread should not have any tasks on it's queue.
2953  // There is a slight possibility that it resumes, steals a task
2954  // from another thread, which spawns more tasks, all in the time
2955  // that it takes this thread to check => don't write an assertion
2956  // that the victim's queue is empty. Try stealing from a
2957  // different thread.
2958  }
2959  } while (asleep);
2960  }
2961 
2962  if (!asleep) {
2963  // We have a victim to try to steal from
2964  task = __kmp_steal_task(other_thread, gtid, task_team,
2965  unfinished_threads, thread_finished,
2966  is_constrained);
2967  }
2968  if (task != NULL) { // set last stolen to victim
2969  if (threads_data[tid].td.td_deque_last_stolen != victim_tid) {
2970  threads_data[tid].td.td_deque_last_stolen = victim_tid;
2971  // The pre-refactored code did not try more than 1 successful new
2972  // vicitm, unless the last one generated more local tasks;
2973  // new_victim keeps track of this
2974  new_victim = 1;
2975  }
2976  } else { // No tasks found; unset last_stolen
2977  KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1);
2978  victim_tid = -2; // no successful victim found
2979  }
2980  }
2981 
2982  if (task == NULL)
2983  break; // break out of tasking loop
2984 
2985 // Found a task; execute it
2986 #if USE_ITT_BUILD && USE_ITT_NOTIFY
2987  if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
2988  if (itt_sync_obj == NULL) { // we are at fork barrier where we could not
2989  // get the object reliably
2990  itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
2991  }
2992  __kmp_itt_task_starting(itt_sync_obj);
2993  }
2994 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
2995  __kmp_invoke_task(gtid, task, current_task);
2996 #if USE_ITT_BUILD
2997  if (itt_sync_obj != NULL)
2998  __kmp_itt_task_finished(itt_sync_obj);
2999 #endif /* USE_ITT_BUILD */
3000  // If this thread is only partway through the barrier and the condition is
3001  // met, then return now, so that the barrier gather/release pattern can
3002  // proceed. If this thread is in the last spin loop in the barrier,
3003  // waiting to be released, we know that the termination condition will not
3004  // be satisfied, so don't waste any cycles checking it.
3005  if (flag == NULL || (!final_spin && flag->done_check())) {
3006  KA_TRACE(
3007  15,
3008  ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3009  gtid));
3010  return TRUE;
3011  }
3012  if (thread->th.th_task_team == NULL) {
3013  break;
3014  }
3015  KMP_YIELD(__kmp_library == library_throughput); // Yield before next task
3016  // If execution of a stolen task results in more tasks being placed on our
3017  // run queue, reset use_own_tasks
3018  if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) {
3019  KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned "
3020  "other tasks, restart\n",
3021  gtid));
3022  use_own_tasks = 1;
3023  new_victim = 0;
3024  }
3025  }
3026 
3027  // The task source has been exhausted. If in final spin loop of barrier,
3028  // check if termination condition is satisfied. The work queue may be empty
3029  // but there might be proxy tasks still executing.
3030  if (final_spin &&
3031  KMP_ATOMIC_LD_ACQ(&current_task->td_incomplete_child_tasks) == 0) {
3032  // First, decrement the #unfinished threads, if that has not already been
3033  // done. This decrement might be to the spin location, and result in the
3034  // termination condition being satisfied.
3035  if (!*thread_finished) {
3036  kmp_int32 count;
3037 
3038  count = KMP_ATOMIC_DEC(unfinished_threads) - 1;
3039  KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec "
3040  "unfinished_threads to %d task_team=%p\n",
3041  gtid, count, task_team));
3042  *thread_finished = TRUE;
3043  }
3044 
3045  // It is now unsafe to reference thread->th.th_team !!!
3046  // Decrementing task_team->tt.tt_unfinished_threads can allow the primary
3047  // thread to pass through the barrier, where it might reset each thread's
3048  // th.th_team field for the next parallel region. If we can steal more
3049  // work, we know that this has not happened yet.
3050  if (flag != NULL && flag->done_check()) {
3051  KA_TRACE(
3052  15,
3053  ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3054  gtid));
3055  return TRUE;
3056  }
3057  }
3058 
3059  // If this thread's task team is NULL, primary thread has recognized that
3060  // there are no more tasks; bail out
3061  if (thread->th.th_task_team == NULL) {
3062  KA_TRACE(15,
3063  ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid));
3064  return FALSE;
3065  }
3066 
3067  // We could be getting tasks from target constructs; if this is the only
3068  // thread, keep trying to execute tasks from own queue
3069  if (nthreads == 1 &&
3070  KMP_ATOMIC_LD_ACQ(&current_task->td_incomplete_child_tasks))
3071  use_own_tasks = 1;
3072  else {
3073  KA_TRACE(15,
3074  ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid));
3075  return FALSE;
3076  }
3077  }
3078 }
3079 
3080 template <bool C, bool S>
3081 int __kmp_execute_tasks_32(
3082  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32<C, S> *flag, int final_spin,
3083  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3084  kmp_int32 is_constrained) {
3085  return __kmp_execute_tasks_template(
3086  thread, gtid, flag, final_spin,
3087  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3088 }
3089 
3090 template <bool C, bool S>
3091 int __kmp_execute_tasks_64(
3092  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64<C, S> *flag, int final_spin,
3093  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3094  kmp_int32 is_constrained) {
3095  return __kmp_execute_tasks_template(
3096  thread, gtid, flag, final_spin,
3097  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3098 }
3099 
3100 int __kmp_execute_tasks_oncore(
3101  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin,
3102  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3103  kmp_int32 is_constrained) {
3104  return __kmp_execute_tasks_template(
3105  thread, gtid, flag, final_spin,
3106  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3107 }
3108 
3109 template int
3110 __kmp_execute_tasks_32<false, false>(kmp_info_t *, kmp_int32,
3111  kmp_flag_32<false, false> *, int,
3112  int *USE_ITT_BUILD_ARG(void *), kmp_int32);
3113 
3114 template int __kmp_execute_tasks_64<false, true>(kmp_info_t *, kmp_int32,
3115  kmp_flag_64<false, true> *,
3116  int,
3117  int *USE_ITT_BUILD_ARG(void *),
3118  kmp_int32);
3119 
3120 template int __kmp_execute_tasks_64<true, false>(kmp_info_t *, kmp_int32,
3121  kmp_flag_64<true, false> *,
3122  int,
3123  int *USE_ITT_BUILD_ARG(void *),
3124  kmp_int32);
3125 
3126 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the
3127 // next barrier so they can assist in executing enqueued tasks.
3128 // First thread in allocates the task team atomically.
3129 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
3130  kmp_info_t *this_thr) {
3131  kmp_thread_data_t *threads_data;
3132  int nthreads, i, is_init_thread;
3133 
3134  KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n",
3135  __kmp_gtid_from_thread(this_thr)));
3136 
3137  KMP_DEBUG_ASSERT(task_team != NULL);
3138  KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL);
3139 
3140  nthreads = task_team->tt.tt_nproc;
3141  KMP_DEBUG_ASSERT(nthreads > 0);
3142  KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc);
3143 
3144  // Allocate or increase the size of threads_data if necessary
3145  is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team);
3146 
3147  if (!is_init_thread) {
3148  // Some other thread already set up the array.
3149  KA_TRACE(
3150  20,
3151  ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
3152  __kmp_gtid_from_thread(this_thr)));
3153  return;
3154  }
3155  threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
3156  KMP_DEBUG_ASSERT(threads_data != NULL);
3157 
3158  if (__kmp_tasking_mode == tskm_task_teams &&
3159  (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) {
3160  // Release any threads sleeping at the barrier, so that they can steal
3161  // tasks and execute them. In extra barrier mode, tasks do not sleep
3162  // at the separate tasking barrier, so this isn't a problem.
3163  for (i = 0; i < nthreads; i++) {
3164  volatile void *sleep_loc;
3165  kmp_info_t *thread = threads_data[i].td.td_thr;
3166 
3167  if (i == this_thr->th.th_info.ds.ds_tid) {
3168  continue;
3169  }
3170  // Since we haven't locked the thread's suspend mutex lock at this
3171  // point, there is a small window where a thread might be putting
3172  // itself to sleep, but hasn't set the th_sleep_loc field yet.
3173  // To work around this, __kmp_execute_tasks_template() periodically checks
3174  // see if other threads are sleeping (using the same random mechanism that
3175  // is used for task stealing) and awakens them if they are.
3176  if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) !=
3177  NULL) {
3178  KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n",
3179  __kmp_gtid_from_thread(this_thr),
3180  __kmp_gtid_from_thread(thread)));
3181  __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc);
3182  } else {
3183  KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",
3184  __kmp_gtid_from_thread(this_thr),
3185  __kmp_gtid_from_thread(thread)));
3186  }
3187  }
3188  }
3189 
3190  KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n",
3191  __kmp_gtid_from_thread(this_thr)));
3192 }
3193 
3194 /* // TODO: Check the comment consistency
3195  * Utility routines for "task teams". A task team (kmp_task_t) is kind of
3196  * like a shadow of the kmp_team_t data struct, with a different lifetime.
3197  * After a child * thread checks into a barrier and calls __kmp_release() from
3198  * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no
3199  * longer assume that the kmp_team_t structure is intact (at any moment, the
3200  * primary thread may exit the barrier code and free the team data structure,
3201  * and return the threads to the thread pool).
3202  *
3203  * This does not work with the tasking code, as the thread is still
3204  * expected to participate in the execution of any tasks that may have been
3205  * spawned my a member of the team, and the thread still needs access to all
3206  * to each thread in the team, so that it can steal work from it.
3207  *
3208  * Enter the existence of the kmp_task_team_t struct. It employs a reference
3209  * counting mechanism, and is allocated by the primary thread before calling
3210  * __kmp_<barrier_kind>_release, and then is release by the last thread to
3211  * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes
3212  * of the kmp_task_team_t structs for consecutive barriers can overlap
3213  * (and will, unless the primary thread is the last thread to exit the barrier
3214  * release phase, which is not typical). The existence of such a struct is
3215  * useful outside the context of tasking.
3216  *
3217  * We currently use the existence of the threads array as an indicator that
3218  * tasks were spawned since the last barrier. If the structure is to be
3219  * useful outside the context of tasking, then this will have to change, but
3220  * not setting the field minimizes the performance impact of tasking on
3221  * barriers, when no explicit tasks were spawned (pushed, actually).
3222  */
3223 
3224 static kmp_task_team_t *__kmp_free_task_teams =
3225  NULL; // Free list for task_team data structures
3226 // Lock for task team data structures
3227 kmp_bootstrap_lock_t __kmp_task_team_lock =
3228  KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock);
3229 
3230 // __kmp_alloc_task_deque:
3231 // Allocates a task deque for a particular thread, and initialize the necessary
3232 // data structures relating to the deque. This only happens once per thread
3233 // per task team since task teams are recycled. No lock is needed during
3234 // allocation since each thread allocates its own deque.
3235 static void __kmp_alloc_task_deque(kmp_info_t *thread,
3236  kmp_thread_data_t *thread_data) {
3237  __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock);
3238  KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL);
3239 
3240  // Initialize last stolen task field to "none"
3241  thread_data->td.td_deque_last_stolen = -1;
3242 
3243  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0);
3244  KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0);
3245  KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0);
3246 
3247  KE_TRACE(
3248  10,
3249  ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",
3250  __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data));
3251  // Allocate space for task deque, and zero the deque
3252  // Cannot use __kmp_thread_calloc() because threads not around for
3253  // kmp_reap_task_team( ).
3254  thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate(
3255  INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *));
3256  thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE;
3257 }
3258 
3259 // __kmp_free_task_deque:
3260 // Deallocates a task deque for a particular thread. Happens at library
3261 // deallocation so don't need to reset all thread data fields.
3262 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) {
3263  if (thread_data->td.td_deque != NULL) {
3264  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3265  TCW_4(thread_data->td.td_deque_ntasks, 0);
3266  __kmp_free(thread_data->td.td_deque);
3267  thread_data->td.td_deque = NULL;
3268  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3269  }
3270 
3271 #ifdef BUILD_TIED_TASK_STACK
3272  // GEH: Figure out what to do here for td_susp_tied_tasks
3273  if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) {
3274  __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data);
3275  }
3276 #endif // BUILD_TIED_TASK_STACK
3277 }
3278 
3279 // __kmp_realloc_task_threads_data:
3280 // Allocates a threads_data array for a task team, either by allocating an
3281 // initial array or enlarging an existing array. Only the first thread to get
3282 // the lock allocs or enlarges the array and re-initializes the array elements.
3283 // That thread returns "TRUE", the rest return "FALSE".
3284 // Assumes that the new array size is given by task_team -> tt.tt_nproc.
3285 // The current size is given by task_team -> tt.tt_max_threads.
3286 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
3287  kmp_task_team_t *task_team) {
3288  kmp_thread_data_t **threads_data_p;
3289  kmp_int32 nthreads, maxthreads;
3290  int is_init_thread = FALSE;
3291 
3292  if (TCR_4(task_team->tt.tt_found_tasks)) {
3293  // Already reallocated and initialized.
3294  return FALSE;
3295  }
3296 
3297  threads_data_p = &task_team->tt.tt_threads_data;
3298  nthreads = task_team->tt.tt_nproc;
3299  maxthreads = task_team->tt.tt_max_threads;
3300 
3301  // All threads must lock when they encounter the first task of the implicit
3302  // task region to make sure threads_data fields are (re)initialized before
3303  // used.
3304  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
3305 
3306  if (!TCR_4(task_team->tt.tt_found_tasks)) {
3307  // first thread to enable tasking
3308  kmp_team_t *team = thread->th.th_team;
3309  int i;
3310 
3311  is_init_thread = TRUE;
3312  if (maxthreads < nthreads) {
3313 
3314  if (*threads_data_p != NULL) {
3315  kmp_thread_data_t *old_data = *threads_data_p;
3316  kmp_thread_data_t *new_data = NULL;
3317 
3318  KE_TRACE(
3319  10,
3320  ("__kmp_realloc_task_threads_data: T#%d reallocating "
3321  "threads data for task_team %p, new_size = %d, old_size = %d\n",
3322  __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads));
3323  // Reallocate threads_data to have more elements than current array
3324  // Cannot use __kmp_thread_realloc() because threads not around for
3325  // kmp_reap_task_team( ). Note all new array entries are initialized
3326  // to zero by __kmp_allocate().
3327  new_data = (kmp_thread_data_t *)__kmp_allocate(
3328  nthreads * sizeof(kmp_thread_data_t));
3329  // copy old data to new data
3330  KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t),
3331  (void *)old_data, maxthreads * sizeof(kmp_thread_data_t));
3332 
3333 #ifdef BUILD_TIED_TASK_STACK
3334  // GEH: Figure out if this is the right thing to do
3335  for (i = maxthreads; i < nthreads; i++) {
3336  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
3337  __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data);
3338  }
3339 #endif // BUILD_TIED_TASK_STACK
3340  // Install the new data and free the old data
3341  (*threads_data_p) = new_data;
3342  __kmp_free(old_data);
3343  } else {
3344  KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating "
3345  "threads data for task_team %p, size = %d\n",
3346  __kmp_gtid_from_thread(thread), task_team, nthreads));
3347  // Make the initial allocate for threads_data array, and zero entries
3348  // Cannot use __kmp_thread_calloc() because threads not around for
3349  // kmp_reap_task_team( ).
3350  *threads_data_p = (kmp_thread_data_t *)__kmp_allocate(
3351  nthreads * sizeof(kmp_thread_data_t));
3352 #ifdef BUILD_TIED_TASK_STACK
3353  // GEH: Figure out if this is the right thing to do
3354  for (i = 0; i < nthreads; i++) {
3355  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
3356  __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data);
3357  }
3358 #endif // BUILD_TIED_TASK_STACK
3359  }
3360  task_team->tt.tt_max_threads = nthreads;
3361  } else {
3362  // If array has (more than) enough elements, go ahead and use it
3363  KMP_DEBUG_ASSERT(*threads_data_p != NULL);
3364  }
3365 
3366  // initialize threads_data pointers back to thread_info structures
3367  for (i = 0; i < nthreads; i++) {
3368  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
3369  thread_data->td.td_thr = team->t.t_threads[i];
3370 
3371  if (thread_data->td.td_deque_last_stolen >= nthreads) {
3372  // The last stolen field survives across teams / barrier, and the number
3373  // of threads may have changed. It's possible (likely?) that a new
3374  // parallel region will exhibit the same behavior as previous region.
3375  thread_data->td.td_deque_last_stolen = -1;
3376  }
3377  }
3378 
3379  KMP_MB();
3380  TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE);
3381  }
3382 
3383  __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
3384  return is_init_thread;
3385 }
3386 
3387 // __kmp_free_task_threads_data:
3388 // Deallocates a threads_data array for a task team, including any attached
3389 // tasking deques. Only occurs at library shutdown.
3390 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) {
3391  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
3392  if (task_team->tt.tt_threads_data != NULL) {
3393  int i;
3394  for (i = 0; i < task_team->tt.tt_max_threads; i++) {
3395  __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]);
3396  }
3397  __kmp_free(task_team->tt.tt_threads_data);
3398  task_team->tt.tt_threads_data = NULL;
3399  }
3400  __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
3401 }
3402 
3403 // __kmp_allocate_task_team:
3404 // Allocates a task team associated with a specific team, taking it from
3405 // the global task team free list if possible. Also initializes data
3406 // structures.
3407 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread,
3408  kmp_team_t *team) {
3409  kmp_task_team_t *task_team = NULL;
3410  int nthreads;
3411 
3412  KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n",
3413  (thread ? __kmp_gtid_from_thread(thread) : -1), team));
3414 
3415  if (TCR_PTR(__kmp_free_task_teams) != NULL) {
3416  // Take a task team from the task team pool
3417  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3418  if (__kmp_free_task_teams != NULL) {
3419  task_team = __kmp_free_task_teams;
3420  TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next);
3421  task_team->tt.tt_next = NULL;
3422  }
3423  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3424  }
3425 
3426  if (task_team == NULL) {
3427  KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating "
3428  "task team for team %p\n",
3429  __kmp_gtid_from_thread(thread), team));
3430  // Allocate a new task team if one is not available. Cannot use
3431  // __kmp_thread_malloc because threads not around for kmp_reap_task_team.
3432  task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t));
3433  __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock);
3434 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG
3435  // suppress race conditions detection on synchronization flags in debug mode
3436  // this helps to analyze library internals eliminating false positives
3437  __itt_suppress_mark_range(
3438  __itt_suppress_range, __itt_suppress_threading_errors,
3439  &task_team->tt.tt_found_tasks, sizeof(task_team->tt.tt_found_tasks));
3440  __itt_suppress_mark_range(__itt_suppress_range,
3441  __itt_suppress_threading_errors,
3442  CCAST(kmp_uint32 *, &task_team->tt.tt_active),
3443  sizeof(task_team->tt.tt_active));
3444 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */
3445  // Note: __kmp_allocate zeroes returned memory, othewise we would need:
3446  // task_team->tt.tt_threads_data = NULL;
3447  // task_team->tt.tt_max_threads = 0;
3448  // task_team->tt.tt_next = NULL;
3449  }
3450 
3451  TCW_4(task_team->tt.tt_found_tasks, FALSE);
3452  TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3453  task_team->tt.tt_nproc = nthreads = team->t.t_nproc;
3454 
3455  KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads);
3456  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
3457  TCW_4(task_team->tt.tt_active, TRUE);
3458 
3459  KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
3460  "unfinished_threads init'd to %d\n",
3461  (thread ? __kmp_gtid_from_thread(thread) : -1), task_team,
3462  KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads)));
3463  return task_team;
3464 }
3465 
3466 // __kmp_free_task_team:
3467 // Frees the task team associated with a specific thread, and adds it
3468 // to the global task team free list.
3469 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) {
3470  KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n",
3471  thread ? __kmp_gtid_from_thread(thread) : -1, task_team));
3472 
3473  // Put task team back on free list
3474  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3475 
3476  KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL);
3477  task_team->tt.tt_next = __kmp_free_task_teams;
3478  TCW_PTR(__kmp_free_task_teams, task_team);
3479 
3480  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3481 }
3482 
3483 // __kmp_reap_task_teams:
3484 // Free all the task teams on the task team free list.
3485 // Should only be done during library shutdown.
3486 // Cannot do anything that needs a thread structure or gtid since they are
3487 // already gone.
3488 void __kmp_reap_task_teams(void) {
3489  kmp_task_team_t *task_team;
3490 
3491  if (TCR_PTR(__kmp_free_task_teams) != NULL) {
3492  // Free all task_teams on the free list
3493  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3494  while ((task_team = __kmp_free_task_teams) != NULL) {
3495  __kmp_free_task_teams = task_team->tt.tt_next;
3496  task_team->tt.tt_next = NULL;
3497 
3498  // Free threads_data if necessary
3499  if (task_team->tt.tt_threads_data != NULL) {
3500  __kmp_free_task_threads_data(task_team);
3501  }
3502  __kmp_free(task_team);
3503  }
3504  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3505  }
3506 }
3507 
3508 // __kmp_wait_to_unref_task_teams:
3509 // Some threads could still be in the fork barrier release code, possibly
3510 // trying to steal tasks. Wait for each thread to unreference its task team.
3511 void __kmp_wait_to_unref_task_teams(void) {
3512  kmp_info_t *thread;
3513  kmp_uint32 spins;
3514  int done;
3515 
3516  KMP_INIT_YIELD(spins);
3517 
3518  for (;;) {
3519  done = TRUE;
3520 
3521  // TODO: GEH - this may be is wrong because some sync would be necessary
3522  // in case threads are added to the pool during the traversal. Need to
3523  // verify that lock for thread pool is held when calling this routine.
3524  for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL;
3525  thread = thread->th.th_next_pool) {
3526 #if KMP_OS_WINDOWS
3527  DWORD exit_val;
3528 #endif
3529  if (TCR_PTR(thread->th.th_task_team) == NULL) {
3530  KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
3531  __kmp_gtid_from_thread(thread)));
3532  continue;
3533  }
3534 #if KMP_OS_WINDOWS
3535  // TODO: GEH - add this check for Linux* OS / OS X* as well?
3536  if (!__kmp_is_thread_alive(thread, &exit_val)) {
3537  thread->th.th_task_team = NULL;
3538  continue;
3539  }
3540 #endif
3541 
3542  done = FALSE; // Because th_task_team pointer is not NULL for this thread
3543 
3544  KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to "
3545  "unreference task_team\n",
3546  __kmp_gtid_from_thread(thread)));
3547 
3548  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
3549  volatile void *sleep_loc;
3550  // If the thread is sleeping, awaken it.
3551  if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) !=
3552  NULL) {
3553  KA_TRACE(
3554  10,
3555  ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",
3556  __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread)));
3557  __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc);
3558  }
3559  }
3560  }
3561  if (done) {
3562  break;
3563  }
3564 
3565  // If oversubscribed or have waited a bit, yield.
3566  KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
3567  }
3568 }
3569 
3570 // __kmp_task_team_setup: Create a task_team for the current team, but use
3571 // an already created, unused one if it already exists.
3572 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
3573  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3574 
3575  // If this task_team hasn't been created yet, allocate it. It will be used in
3576  // the region after the next.
3577  // If it exists, it is the current task team and shouldn't be touched yet as
3578  // it may still be in use.
3579  if (team->t.t_task_team[this_thr->th.th_task_state] == NULL &&
3580  (always || team->t.t_nproc > 1)) {
3581  team->t.t_task_team[this_thr->th.th_task_state] =
3582  __kmp_allocate_task_team(this_thr, team);
3583  KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created new task_team %p"
3584  " for team %d at parity=%d\n",
3585  __kmp_gtid_from_thread(this_thr),
3586  team->t.t_task_team[this_thr->th.th_task_state], team->t.t_id,
3587  this_thr->th.th_task_state));
3588  }
3589 
3590  // After threads exit the release, they will call sync, and then point to this
3591  // other task_team; make sure it is allocated and properly initialized. As
3592  // threads spin in the barrier release phase, they will continue to use the
3593  // previous task_team struct(above), until they receive the signal to stop
3594  // checking for tasks (they can't safely reference the kmp_team_t struct,
3595  // which could be reallocated by the primary thread). No task teams are formed
3596  // for serialized teams.
3597  if (team->t.t_nproc > 1) {
3598  int other_team = 1 - this_thr->th.th_task_state;
3599  KMP_DEBUG_ASSERT(other_team >= 0 && other_team < 2);
3600  if (team->t.t_task_team[other_team] == NULL) { // setup other team as well
3601  team->t.t_task_team[other_team] =
3602  __kmp_allocate_task_team(this_thr, team);
3603  KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created second new "
3604  "task_team %p for team %d at parity=%d\n",
3605  __kmp_gtid_from_thread(this_thr),
3606  team->t.t_task_team[other_team], team->t.t_id, other_team));
3607  } else { // Leave the old task team struct in place for the upcoming region;
3608  // adjust as needed
3609  kmp_task_team_t *task_team = team->t.t_task_team[other_team];
3610  if (!task_team->tt.tt_active ||
3611  team->t.t_nproc != task_team->tt.tt_nproc) {
3612  TCW_4(task_team->tt.tt_nproc, team->t.t_nproc);
3613  TCW_4(task_team->tt.tt_found_tasks, FALSE);
3614  TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3615  KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads,
3616  team->t.t_nproc);
3617  TCW_4(task_team->tt.tt_active, TRUE);
3618  }
3619  // if team size has changed, the first thread to enable tasking will
3620  // realloc threads_data if necessary
3621  KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d reset next task_team "
3622  "%p for team %d at parity=%d\n",
3623  __kmp_gtid_from_thread(this_thr),
3624  team->t.t_task_team[other_team], team->t.t_id, other_team));
3625  }
3626  }
3627 
3628  // For regular thread, task enabling should be called when the task is going
3629  // to be pushed to a dequeue. However, for the hidden helper thread, we need
3630  // it ahead of time so that some operations can be performed without race
3631  // condition.
3632  if (this_thr == __kmp_hidden_helper_main_thread) {
3633  for (int i = 0; i < 2; ++i) {
3634  kmp_task_team_t *task_team = team->t.t_task_team[i];
3635  if (KMP_TASKING_ENABLED(task_team)) {
3636  continue;
3637  }
3638  __kmp_enable_tasking(task_team, this_thr);
3639  for (int j = 0; j < task_team->tt.tt_nproc; ++j) {
3640  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[j];
3641  if (thread_data->td.td_deque == NULL) {
3642  __kmp_alloc_task_deque(__kmp_hidden_helper_threads[j], thread_data);
3643  }
3644  }
3645  }
3646  }
3647 }
3648 
3649 // __kmp_task_team_sync: Propagation of task team data from team to threads
3650 // which happens just after the release phase of a team barrier. This may be
3651 // called by any thread, but only for teams with # threads > 1.
3652 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
3653  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3654 
3655  // Toggle the th_task_state field, to switch which task_team this thread
3656  // refers to
3657  this_thr->th.th_task_state = (kmp_uint8)(1 - this_thr->th.th_task_state);
3658 
3659  // It is now safe to propagate the task team pointer from the team struct to
3660  // the current thread.
3661  TCW_PTR(this_thr->th.th_task_team,
3662  team->t.t_task_team[this_thr->th.th_task_state]);
3663  KA_TRACE(20,
3664  ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
3665  "%p from Team #%d (parity=%d)\n",
3666  __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team,
3667  team->t.t_id, this_thr->th.th_task_state));
3668 }
3669 
3670 // __kmp_task_team_wait: Primary thread waits for outstanding tasks after the
3671 // barrier gather phase. Only called by primary thread if #threads in team > 1
3672 // or if proxy tasks were created.
3673 //
3674 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off
3675 // by passing in 0 optionally as the last argument. When wait is zero, primary
3676 // thread does not wait for unfinished_threads to reach 0.
3677 void __kmp_task_team_wait(
3678  kmp_info_t *this_thr,
3679  kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) {
3680  kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state];
3681 
3682  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3683  KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team);
3684 
3685  if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) {
3686  if (wait) {
3687  KA_TRACE(20, ("__kmp_task_team_wait: Primary T#%d waiting for all tasks "
3688  "(for unfinished_threads to reach 0) on task_team = %p\n",
3689  __kmp_gtid_from_thread(this_thr), task_team));
3690  // Worker threads may have dropped through to release phase, but could
3691  // still be executing tasks. Wait here for tasks to complete. To avoid
3692  // memory contention, only primary thread checks termination condition.
3693  kmp_flag_32<false, false> flag(
3694  RCAST(std::atomic<kmp_uint32> *,
3695  &task_team->tt.tt_unfinished_threads),
3696  0U);
3697  flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
3698  }
3699  // Deactivate the old task team, so that the worker threads will stop
3700  // referencing it while spinning.
3701  KA_TRACE(
3702  20,
3703  ("__kmp_task_team_wait: Primary T#%d deactivating task_team %p: "
3704  "setting active to false, setting local and team's pointer to NULL\n",
3705  __kmp_gtid_from_thread(this_thr), task_team));
3706  KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 ||
3707  task_team->tt.tt_found_proxy_tasks == TRUE);
3708  TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3709  KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0);
3710  TCW_SYNC_4(task_team->tt.tt_active, FALSE);
3711  KMP_MB();
3712 
3713  TCW_PTR(this_thr->th.th_task_team, NULL);
3714  }
3715 }
3716 
3717 // __kmp_tasking_barrier:
3718 // This routine is called only when __kmp_tasking_mode == tskm_extra_barrier.
3719 // Internal function to execute all tasks prior to a regular barrier or a join
3720 // barrier. It is a full barrier itself, which unfortunately turns regular
3721 // barriers into double barriers and join barriers into 1 1/2 barriers.
3722 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) {
3723  std::atomic<kmp_uint32> *spin = RCAST(
3724  std::atomic<kmp_uint32> *,
3725  &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads);
3726  int flag = FALSE;
3727  KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier);
3728 
3729 #if USE_ITT_BUILD
3730  KMP_FSYNC_SPIN_INIT(spin, NULL);
3731 #endif /* USE_ITT_BUILD */
3732  kmp_flag_32<false, false> spin_flag(spin, 0U);
3733  while (!spin_flag.execute_tasks(thread, gtid, TRUE,
3734  &flag USE_ITT_BUILD_ARG(NULL), 0)) {
3735 #if USE_ITT_BUILD
3736  // TODO: What about itt_sync_obj??
3737  KMP_FSYNC_SPIN_PREPARE(RCAST(void *, spin));
3738 #endif /* USE_ITT_BUILD */
3739 
3740  if (TCR_4(__kmp_global.g.g_done)) {
3741  if (__kmp_global.g.g_abort)
3742  __kmp_abort_thread();
3743  break;
3744  }
3745  KMP_YIELD(TRUE);
3746  }
3747 #if USE_ITT_BUILD
3748  KMP_FSYNC_SPIN_ACQUIRED(RCAST(void *, spin));
3749 #endif /* USE_ITT_BUILD */
3750 }
3751 
3752 // __kmp_give_task puts a task into a given thread queue if:
3753 // - the queue for that thread was created
3754 // - there's space in that queue
3755 // Because of this, __kmp_push_task needs to check if there's space after
3756 // getting the lock
3757 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task,
3758  kmp_int32 pass) {
3759  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
3760  kmp_task_team_t *task_team = taskdata->td_task_team;
3761 
3762  KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n",
3763  taskdata, tid));
3764 
3765  // If task_team is NULL something went really bad...
3766  KMP_DEBUG_ASSERT(task_team != NULL);
3767 
3768  bool result = false;
3769  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
3770 
3771  if (thread_data->td.td_deque == NULL) {
3772  // There's no queue in this thread, go find another one
3773  // We're guaranteed that at least one thread has a queue
3774  KA_TRACE(30,
3775  ("__kmp_give_task: thread %d has no queue while giving task %p.\n",
3776  tid, taskdata));
3777  return result;
3778  }
3779 
3780  if (TCR_4(thread_data->td.td_deque_ntasks) >=
3781  TASK_DEQUE_SIZE(thread_data->td)) {
3782  KA_TRACE(
3783  30,
3784  ("__kmp_give_task: queue is full while giving task %p to thread %d.\n",
3785  taskdata, tid));
3786 
3787  // if this deque is bigger than the pass ratio give a chance to another
3788  // thread
3789  if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
3790  return result;
3791 
3792  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3793  if (TCR_4(thread_data->td.td_deque_ntasks) >=
3794  TASK_DEQUE_SIZE(thread_data->td)) {
3795  // expand deque to push the task which is not allowed to execute
3796  __kmp_realloc_task_deque(thread, thread_data);
3797  }
3798 
3799  } else {
3800 
3801  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3802 
3803  if (TCR_4(thread_data->td.td_deque_ntasks) >=
3804  TASK_DEQUE_SIZE(thread_data->td)) {
3805  KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to "
3806  "thread %d.\n",
3807  taskdata, tid));
3808 
3809  // if this deque is bigger than the pass ratio give a chance to another
3810  // thread
3811  if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
3812  goto release_and_exit;
3813 
3814  __kmp_realloc_task_deque(thread, thread_data);
3815  }
3816  }
3817 
3818  // lock is held here, and there is space in the deque
3819 
3820  thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata;
3821  // Wrap index.
3822  thread_data->td.td_deque_tail =
3823  (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
3824  TCW_4(thread_data->td.td_deque_ntasks,
3825  TCR_4(thread_data->td.td_deque_ntasks) + 1);
3826 
3827  result = true;
3828  KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n",
3829  taskdata, tid));
3830 
3831 release_and_exit:
3832  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3833 
3834  return result;
3835 }
3836 
3837 #define PROXY_TASK_FLAG 0x40000000
3838 /* The finish of the proxy tasks is divided in two pieces:
3839  - the top half is the one that can be done from a thread outside the team
3840  - the bottom half must be run from a thread within the team
3841 
3842  In order to run the bottom half the task gets queued back into one of the
3843  threads of the team. Once the td_incomplete_child_task counter of the parent
3844  is decremented the threads can leave the barriers. So, the bottom half needs
3845  to be queued before the counter is decremented. The top half is therefore
3846  divided in two parts:
3847  - things that can be run before queuing the bottom half
3848  - things that must be run after queuing the bottom half
3849 
3850  This creates a second race as the bottom half can free the task before the
3851  second top half is executed. To avoid this we use the
3852  td_incomplete_child_task of the proxy task to synchronize the top and bottom
3853  half. */
3854 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
3855  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
3856  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3857  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
3858  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
3859 
3860  taskdata->td_flags.complete = 1; // mark the task as completed
3861 
3862  if (taskdata->td_taskgroup)
3863  KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
3864 
3865  // Create an imaginary children for this task so the bottom half cannot
3866  // release the task before we have completed the second top half
3867  KMP_ATOMIC_OR(&taskdata->td_incomplete_child_tasks, PROXY_TASK_FLAG);
3868 }
3869 
3870 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
3871  kmp_int32 children = 0;
3872 
3873  // Predecrement simulated by "- 1" calculation
3874  children =
3875  KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1;
3876  KMP_DEBUG_ASSERT(children >= 0);
3877 
3878  // Remove the imaginary children
3879  KMP_ATOMIC_AND(&taskdata->td_incomplete_child_tasks, ~PROXY_TASK_FLAG);
3880 }
3881 
3882 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) {
3883  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3884  kmp_info_t *thread = __kmp_threads[gtid];
3885 
3886  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3887  KMP_DEBUG_ASSERT(taskdata->td_flags.complete ==
3888  1); // top half must run before bottom half
3889 
3890  // We need to wait to make sure the top half is finished
3891  // Spinning here should be ok as this should happen quickly
3892  while ((KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) &
3893  PROXY_TASK_FLAG) > 0)
3894  ;
3895 
3896  __kmp_release_deps(gtid, taskdata);
3897  __kmp_free_task_and_ancestors(gtid, taskdata, thread);
3898 }
3899 
3908 void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) {
3909  KMP_DEBUG_ASSERT(ptask != NULL);
3910  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3911  KA_TRACE(
3912  10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n",
3913  gtid, taskdata));
3914  __kmp_assert_valid_gtid(gtid);
3915  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3916 
3917  __kmp_first_top_half_finish_proxy(taskdata);
3918  __kmp_second_top_half_finish_proxy(taskdata);
3919  __kmp_bottom_half_finish_proxy(gtid, ptask);
3920 
3921  KA_TRACE(10,
3922  ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n",
3923  gtid, taskdata));
3924 }
3925 
3926 void __kmpc_give_task(kmp_task_t *ptask, kmp_int32 start = 0) {
3927  KMP_DEBUG_ASSERT(ptask != NULL);
3928  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3929 
3930  // Enqueue task to complete bottom half completion from a thread within the
3931  // corresponding team
3932  kmp_team_t *team = taskdata->td_team;
3933  kmp_int32 nthreads = team->t.t_nproc;
3934  kmp_info_t *thread;
3935 
3936  // This should be similar to start_k = __kmp_get_random( thread ) % nthreads
3937  // but we cannot use __kmp_get_random here
3938  kmp_int32 start_k = start;
3939  kmp_int32 pass = 1;
3940  kmp_int32 k = start_k;
3941 
3942  do {
3943  // For now we're just linearly trying to find a thread
3944  thread = team->t.t_threads[k];
3945  k = (k + 1) % nthreads;
3946 
3947  // we did a full pass through all the threads
3948  if (k == start_k)
3949  pass = pass << 1;
3950 
3951  } while (!__kmp_give_task(thread, k, ptask, pass));
3952 }
3953 
3961 void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) {
3962  KMP_DEBUG_ASSERT(ptask != NULL);
3963  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3964 
3965  KA_TRACE(
3966  10,
3967  ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n",
3968  taskdata));
3969 
3970  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3971 
3972  __kmp_first_top_half_finish_proxy(taskdata);
3973 
3974  __kmpc_give_task(ptask);
3975 
3976  __kmp_second_top_half_finish_proxy(taskdata);
3977 
3978  KA_TRACE(
3979  10,
3980  ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n",
3981  taskdata));
3982 }
3983 
3984 kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref, int gtid,
3985  kmp_task_t *task) {
3986  kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
3987  if (td->td_allow_completion_event.type == KMP_EVENT_UNINITIALIZED) {
3988  td->td_allow_completion_event.type = KMP_EVENT_ALLOW_COMPLETION;
3989  td->td_allow_completion_event.ed.task = task;
3990  __kmp_init_tas_lock(&td->td_allow_completion_event.lock);
3991  }
3992  return &td->td_allow_completion_event;
3993 }
3994 
3995 void __kmp_fulfill_event(kmp_event_t *event) {
3996  if (event->type == KMP_EVENT_ALLOW_COMPLETION) {
3997  kmp_task_t *ptask = event->ed.task;
3998  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3999  bool detached = false;
4000  int gtid = __kmp_get_gtid();
4001 
4002  // The associated task might have completed or could be completing at this
4003  // point.
4004  // We need to take the lock to avoid races
4005  __kmp_acquire_tas_lock(&event->lock, gtid);
4006  if (taskdata->td_flags.proxy == TASK_PROXY) {
4007  detached = true;
4008  } else {
4009 #if OMPT_SUPPORT
4010  // The OMPT event must occur under mutual exclusion,
4011  // otherwise the tool might access ptask after free
4012  if (UNLIKELY(ompt_enabled.enabled))
4013  __ompt_task_finish(ptask, NULL, ompt_task_early_fulfill);
4014 #endif
4015  }
4016  event->type = KMP_EVENT_UNINITIALIZED;
4017  __kmp_release_tas_lock(&event->lock, gtid);
4018 
4019  if (detached) {
4020 #if OMPT_SUPPORT
4021  // We free ptask afterwards and know the task is finished,
4022  // so locking is not necessary
4023  if (UNLIKELY(ompt_enabled.enabled))
4024  __ompt_task_finish(ptask, NULL, ompt_task_late_fulfill);
4025 #endif
4026  // If the task detached complete the proxy task
4027  if (gtid >= 0) {
4028  kmp_team_t *team = taskdata->td_team;
4029  kmp_info_t *thread = __kmp_get_thread();
4030  if (thread->th.th_team == team) {
4031  __kmpc_proxy_task_completed(gtid, ptask);
4032  return;
4033  }
4034  }
4035 
4036  // fallback
4038  }
4039  }
4040 }
4041 
4042 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task
4043 // for taskloop
4044 //
4045 // thread: allocating thread
4046 // task_src: pointer to source task to be duplicated
4047 // returns: a pointer to the allocated kmp_task_t structure (task).
4048 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
4049  kmp_task_t *task;
4050  kmp_taskdata_t *taskdata;
4051  kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
4052  kmp_taskdata_t *parent_task = taskdata_src->td_parent; // same parent task
4053  size_t shareds_offset;
4054  size_t task_size;
4055 
4056  KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread,
4057  task_src));
4058  KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy ==
4059  TASK_FULL); // it should not be proxy task
4060  KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT);
4061  task_size = taskdata_src->td_size_alloc;
4062 
4063  // Allocate a kmp_taskdata_t block and a kmp_task_t block.
4064  KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread,
4065  task_size));
4066 #if USE_FAST_MEMORY
4067  taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size);
4068 #else
4069  taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size);
4070 #endif /* USE_FAST_MEMORY */
4071  KMP_MEMCPY(taskdata, taskdata_src, task_size);
4072 
4073  task = KMP_TASKDATA_TO_TASK(taskdata);
4074 
4075  // Initialize new task (only specific fields not affected by memcpy)
4076  taskdata->td_task_id = KMP_GEN_TASK_ID();
4077  if (task->shareds != NULL) { // need setup shareds pointer
4078  shareds_offset = (char *)task_src->shareds - (char *)taskdata_src;
4079  task->shareds = &((char *)taskdata)[shareds_offset];
4080  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
4081  0);
4082  }
4083  taskdata->td_alloc_thread = thread;
4084  taskdata->td_parent = parent_task;
4085  // task inherits the taskgroup from the parent task
4086  taskdata->td_taskgroup = parent_task->td_taskgroup;
4087  // tied task needs to initialize the td_last_tied at creation,
4088  // untied one does this when it is scheduled for execution
4089  if (taskdata->td_flags.tiedness == TASK_TIED)
4090  taskdata->td_last_tied = taskdata;
4091 
4092  // Only need to keep track of child task counts if team parallel and tasking
4093  // not serialized
4094  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
4095  KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
4096  if (parent_task->td_taskgroup)
4097  KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
4098  // Only need to keep track of allocated child tasks for explicit tasks since
4099  // implicit not deallocated
4100  if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT)
4101  KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks);
4102  }
4103 
4104  KA_TRACE(20,
4105  ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n",
4106  thread, taskdata, taskdata->td_parent));
4107 #if OMPT_SUPPORT
4108  if (UNLIKELY(ompt_enabled.enabled))
4109  __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid);
4110 #endif
4111  return task;
4112 }
4113 
4114 // Routine optionally generated by the compiler for setting the lastprivate flag
4115 // and calling needed constructors for private/firstprivate objects
4116 // (used to form taskloop tasks from pattern task)
4117 // Parameters: dest task, src task, lastprivate flag.
4118 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
4119 
4120 KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8);
4121 
4122 // class to encapsulate manipulating loop bounds in a taskloop task.
4123 // this abstracts away the Intel vs GOMP taskloop interface for setting/getting
4124 // the loop bound variables.
4125 class kmp_taskloop_bounds_t {
4126  kmp_task_t *task;
4127  const kmp_taskdata_t *taskdata;
4128  size_t lower_offset;
4129  size_t upper_offset;
4130 
4131 public:
4132  kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub)
4133  : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)),
4134  lower_offset((char *)lb - (char *)task),
4135  upper_offset((char *)ub - (char *)task) {
4136  KMP_DEBUG_ASSERT((char *)lb > (char *)_task);
4137  KMP_DEBUG_ASSERT((char *)ub > (char *)_task);
4138  }
4139  kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds)
4140  : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)),
4141  lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {}
4142  size_t get_lower_offset() const { return lower_offset; }
4143  size_t get_upper_offset() const { return upper_offset; }
4144  kmp_uint64 get_lb() const {
4145  kmp_int64 retval;
4146 #if defined(KMP_GOMP_COMPAT)
4147  // Intel task just returns the lower bound normally
4148  if (!taskdata->td_flags.native) {
4149  retval = *(kmp_int64 *)((char *)task + lower_offset);
4150  } else {
4151  // GOMP task has to take into account the sizeof(long)
4152  if (taskdata->td_size_loop_bounds == 4) {
4153  kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds);
4154  retval = (kmp_int64)*lb;
4155  } else {
4156  kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds);
4157  retval = (kmp_int64)*lb;
4158  }
4159  }
4160 #else
4161  (void)taskdata;
4162  retval = *(kmp_int64 *)((char *)task + lower_offset);
4163 #endif // defined(KMP_GOMP_COMPAT)
4164  return retval;
4165  }
4166  kmp_uint64 get_ub() const {
4167  kmp_int64 retval;
4168 #if defined(KMP_GOMP_COMPAT)
4169  // Intel task just returns the upper bound normally
4170  if (!taskdata->td_flags.native) {
4171  retval = *(kmp_int64 *)((char *)task + upper_offset);
4172  } else {
4173  // GOMP task has to take into account the sizeof(long)
4174  if (taskdata->td_size_loop_bounds == 4) {
4175  kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1;
4176  retval = (kmp_int64)*ub;
4177  } else {
4178  kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1;
4179  retval = (kmp_int64)*ub;
4180  }
4181  }
4182 #else
4183  retval = *(kmp_int64 *)((char *)task + upper_offset);
4184 #endif // defined(KMP_GOMP_COMPAT)
4185  return retval;
4186  }
4187  void set_lb(kmp_uint64 lb) {
4188 #if defined(KMP_GOMP_COMPAT)
4189  // Intel task just sets the lower bound normally
4190  if (!taskdata->td_flags.native) {
4191  *(kmp_uint64 *)((char *)task + lower_offset) = lb;
4192  } else {
4193  // GOMP task has to take into account the sizeof(long)
4194  if (taskdata->td_size_loop_bounds == 4) {
4195  kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds);
4196  *lower = (kmp_uint32)lb;
4197  } else {
4198  kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds);
4199  *lower = (kmp_uint64)lb;
4200  }
4201  }
4202 #else
4203  *(kmp_uint64 *)((char *)task + lower_offset) = lb;
4204 #endif // defined(KMP_GOMP_COMPAT)
4205  }
4206  void set_ub(kmp_uint64 ub) {
4207 #if defined(KMP_GOMP_COMPAT)
4208  // Intel task just sets the upper bound normally
4209  if (!taskdata->td_flags.native) {
4210  *(kmp_uint64 *)((char *)task + upper_offset) = ub;
4211  } else {
4212  // GOMP task has to take into account the sizeof(long)
4213  if (taskdata->td_size_loop_bounds == 4) {
4214  kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1;
4215  *upper = (kmp_uint32)ub;
4216  } else {
4217  kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1;
4218  *upper = (kmp_uint64)ub;
4219  }
4220  }
4221 #else
4222  *(kmp_uint64 *)((char *)task + upper_offset) = ub;
4223 #endif // defined(KMP_GOMP_COMPAT)
4224  }
4225 };
4226 
4227 // __kmp_taskloop_linear: Start tasks of the taskloop linearly
4228 //
4229 // loc Source location information
4230 // gtid Global thread ID
4231 // task Pattern task, exposes the loop iteration range
4232 // lb Pointer to loop lower bound in task structure
4233 // ub Pointer to loop upper bound in task structure
4234 // st Loop stride
4235 // ub_glob Global upper bound (used for lastprivate check)
4236 // num_tasks Number of tasks to execute
4237 // grainsize Number of loop iterations per task
4238 // extras Number of chunks with grainsize+1 iterations
4239 // last_chunk Reduction of grainsize for last task
4240 // tc Iterations count
4241 // task_dup Tasks duplication routine
4242 // codeptr_ra Return address for OMPT events
4243 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
4244  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4245  kmp_uint64 ub_glob, kmp_uint64 num_tasks,
4246  kmp_uint64 grainsize, kmp_uint64 extras,
4247  kmp_int64 last_chunk, kmp_uint64 tc,
4248 #if OMPT_SUPPORT
4249  void *codeptr_ra,
4250 #endif
4251  void *task_dup) {
4252  KMP_COUNT_BLOCK(OMP_TASKLOOP);
4253  KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling);
4254  p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4255  // compiler provides global bounds here
4256  kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4257  kmp_uint64 lower = task_bounds.get_lb();
4258  kmp_uint64 upper = task_bounds.get_ub();
4259  kmp_uint64 i;
4260  kmp_info_t *thread = __kmp_threads[gtid];
4261  kmp_taskdata_t *current_task = thread->th.th_current_task;
4262  kmp_task_t *next_task;
4263  kmp_int32 lastpriv = 0;
4264 
4265  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
4266  (last_chunk < 0 ? last_chunk : extras));
4267  KMP_DEBUG_ASSERT(num_tasks > extras);
4268  KMP_DEBUG_ASSERT(num_tasks > 0);
4269  KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
4270  "extras %lld, last_chunk %lld, i=%lld,%lld(%d)%lld, dup %p\n",
4271  gtid, num_tasks, grainsize, extras, last_chunk, lower, upper,
4272  ub_glob, st, task_dup));
4273 
4274  // Launch num_tasks tasks, assign grainsize iterations each task
4275  for (i = 0; i < num_tasks; ++i) {
4276  kmp_uint64 chunk_minus_1;
4277  if (extras == 0) {
4278  chunk_minus_1 = grainsize - 1;
4279  } else {
4280  chunk_minus_1 = grainsize;
4281  --extras; // first extras iterations get bigger chunk (grainsize+1)
4282  }
4283  upper = lower + st * chunk_minus_1;
4284  if (upper > *ub) {
4285  upper = *ub;
4286  }
4287  if (i == num_tasks - 1) {
4288  // schedule the last task, set lastprivate flag if needed
4289  if (st == 1) { // most common case
4290  KMP_DEBUG_ASSERT(upper == *ub);
4291  if (upper == ub_glob)
4292  lastpriv = 1;
4293  } else if (st > 0) { // positive loop stride
4294  KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper);
4295  if ((kmp_uint64)st > ub_glob - upper)
4296  lastpriv = 1;
4297  } else { // negative loop stride
4298  KMP_DEBUG_ASSERT(upper + st < *ub);
4299  if (upper - ub_glob < (kmp_uint64)(-st))
4300  lastpriv = 1;
4301  }
4302  }
4303  next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
4304  kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
4305  kmp_taskloop_bounds_t next_task_bounds =
4306  kmp_taskloop_bounds_t(next_task, task_bounds);
4307 
4308  // adjust task-specific bounds
4309  next_task_bounds.set_lb(lower);
4310  if (next_taskdata->td_flags.native) {
4311  next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1));
4312  } else {
4313  next_task_bounds.set_ub(upper);
4314  }
4315  if (ptask_dup != NULL) // set lastprivate flag, construct firstprivates,
4316  // etc.
4317  ptask_dup(next_task, task, lastpriv);
4318  KA_TRACE(40,
4319  ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
4320  "upper %lld stride %lld, (offsets %p %p)\n",
4321  gtid, i, next_task, lower, upper, st,
4322  next_task_bounds.get_lower_offset(),
4323  next_task_bounds.get_upper_offset()));
4324 #if OMPT_SUPPORT
4325  __kmp_omp_taskloop_task(NULL, gtid, next_task,
4326  codeptr_ra); // schedule new task
4327 #else
4328  __kmp_omp_task(gtid, next_task, true); // schedule new task
4329 #endif
4330  lower = upper + st; // adjust lower bound for the next iteration
4331  }
4332  // free the pattern task and exit
4333  __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
4334  // do not execute the pattern task, just do internal bookkeeping
4335  __kmp_task_finish<false>(gtid, task, current_task);
4336 }
4337 
4338 // Structure to keep taskloop parameters for auxiliary task
4339 // kept in the shareds of the task structure.
4340 typedef struct __taskloop_params {
4341  kmp_task_t *task;
4342  kmp_uint64 *lb;
4343  kmp_uint64 *ub;
4344  void *task_dup;
4345  kmp_int64 st;
4346  kmp_uint64 ub_glob;
4347  kmp_uint64 num_tasks;
4348  kmp_uint64 grainsize;
4349  kmp_uint64 extras;
4350  kmp_int64 last_chunk;
4351  kmp_uint64 tc;
4352  kmp_uint64 num_t_min;
4353 #if OMPT_SUPPORT
4354  void *codeptr_ra;
4355 #endif
4356 } __taskloop_params_t;
4357 
4358 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *,
4359  kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64,
4360  kmp_uint64, kmp_uint64, kmp_int64, kmp_uint64,
4361  kmp_uint64,
4362 #if OMPT_SUPPORT
4363  void *,
4364 #endif
4365  void *);
4366 
4367 // Execute part of the taskloop submitted as a task.
4368 int __kmp_taskloop_task(int gtid, void *ptask) {
4369  __taskloop_params_t *p =
4370  (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds;
4371  kmp_task_t *task = p->task;
4372  kmp_uint64 *lb = p->lb;
4373  kmp_uint64 *ub = p->ub;
4374  void *task_dup = p->task_dup;
4375  // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4376  kmp_int64 st = p->st;
4377  kmp_uint64 ub_glob = p->ub_glob;
4378  kmp_uint64 num_tasks = p->num_tasks;
4379  kmp_uint64 grainsize = p->grainsize;
4380  kmp_uint64 extras = p->extras;
4381  kmp_int64 last_chunk = p->last_chunk;
4382  kmp_uint64 tc = p->tc;
4383  kmp_uint64 num_t_min = p->num_t_min;
4384 #if OMPT_SUPPORT
4385  void *codeptr_ra = p->codeptr_ra;
4386 #endif
4387 #if KMP_DEBUG
4388  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4389  KMP_DEBUG_ASSERT(task != NULL);
4390  KA_TRACE(20,
4391  ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
4392  " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
4393  gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
4394  st, task_dup));
4395 #endif
4396  KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min);
4397  if (num_tasks > num_t_min)
4398  __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
4399  grainsize, extras, last_chunk, tc, num_t_min,
4400 #if OMPT_SUPPORT
4401  codeptr_ra,
4402 #endif
4403  task_dup);
4404  else
4405  __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
4406  grainsize, extras, last_chunk, tc,
4407 #if OMPT_SUPPORT
4408  codeptr_ra,
4409 #endif
4410  task_dup);
4411 
4412  KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid));
4413  return 0;
4414 }
4415 
4416 // Schedule part of the taskloop as a task,
4417 // execute the rest of the taskloop.
4418 //
4419 // loc Source location information
4420 // gtid Global thread ID
4421 // task Pattern task, exposes the loop iteration range
4422 // lb Pointer to loop lower bound in task structure
4423 // ub Pointer to loop upper bound in task structure
4424 // st Loop stride
4425 // ub_glob Global upper bound (used for lastprivate check)
4426 // num_tasks Number of tasks to execute
4427 // grainsize Number of loop iterations per task
4428 // extras Number of chunks with grainsize+1 iterations
4429 // last_chunk Reduction of grainsize for last task
4430 // tc Iterations count
4431 // num_t_min Threshold to launch tasks recursively
4432 // task_dup Tasks duplication routine
4433 // codeptr_ra Return address for OMPT events
4434 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
4435  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4436  kmp_uint64 ub_glob, kmp_uint64 num_tasks,
4437  kmp_uint64 grainsize, kmp_uint64 extras,
4438  kmp_int64 last_chunk, kmp_uint64 tc,
4439  kmp_uint64 num_t_min,
4440 #if OMPT_SUPPORT
4441  void *codeptr_ra,
4442 #endif
4443  void *task_dup) {
4444  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4445  KMP_DEBUG_ASSERT(task != NULL);
4446  KMP_DEBUG_ASSERT(num_tasks > num_t_min);
4447  KA_TRACE(20,
4448  ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
4449  " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
4450  gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
4451  st, task_dup));
4452  p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4453  kmp_uint64 lower = *lb;
4454  kmp_info_t *thread = __kmp_threads[gtid];
4455  // kmp_taskdata_t *current_task = thread->th.th_current_task;
4456  kmp_task_t *next_task;
4457  size_t lower_offset =
4458  (char *)lb - (char *)task; // remember offset of lb in the task structure
4459  size_t upper_offset =
4460  (char *)ub - (char *)task; // remember offset of ub in the task structure
4461 
4462  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
4463  (last_chunk < 0 ? last_chunk : extras));
4464  KMP_DEBUG_ASSERT(num_tasks > extras);
4465  KMP_DEBUG_ASSERT(num_tasks > 0);
4466 
4467  // split the loop in two halves
4468  kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1;
4469  kmp_int64 last_chunk0 = 0, last_chunk1 = 0;
4470  kmp_uint64 gr_size0 = grainsize;
4471  kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute
4472  kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task
4473  if (last_chunk < 0) {
4474  ext0 = ext1 = 0;
4475  last_chunk1 = last_chunk;
4476  tc0 = grainsize * n_tsk0;
4477  tc1 = tc - tc0;
4478  } else if (n_tsk0 <= extras) {
4479  gr_size0++; // integrate extras into grainsize
4480  ext0 = 0; // no extra iters in 1st half
4481  ext1 = extras - n_tsk0; // remaining extras
4482  tc0 = gr_size0 * n_tsk0;
4483  tc1 = tc - tc0;
4484  } else { // n_tsk0 > extras
4485  ext1 = 0; // no extra iters in 2nd half
4486  ext0 = extras;
4487  tc1 = grainsize * n_tsk1;
4488  tc0 = tc - tc1;
4489  }
4490  ub0 = lower + st * (tc0 - 1);
4491  lb1 = ub0 + st;
4492 
4493  // create pattern task for 2nd half of the loop
4494  next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
4495  // adjust lower bound (upper bound is not changed) for the 2nd half
4496  *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
4497  if (ptask_dup != NULL) // construct firstprivates, etc.
4498  ptask_dup(next_task, task, 0);
4499  *ub = ub0; // adjust upper bound for the 1st half
4500 
4501  // create auxiliary task for 2nd half of the loop
4502  // make sure new task has same parent task as the pattern task
4503  kmp_taskdata_t *current_task = thread->th.th_current_task;
4504  thread->th.th_current_task = taskdata->td_parent;
4505  kmp_task_t *new_task =
4506  __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *),
4507  sizeof(__taskloop_params_t), &__kmp_taskloop_task);
4508  // restore current task
4509  thread->th.th_current_task = current_task;
4510  __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds;
4511  p->task = next_task;
4512  p->lb = (kmp_uint64 *)((char *)next_task + lower_offset);
4513  p->ub = (kmp_uint64 *)((char *)next_task + upper_offset);
4514  p->task_dup = task_dup;
4515  p->st = st;
4516  p->ub_glob = ub_glob;
4517  p->num_tasks = n_tsk1;
4518  p->grainsize = grainsize;
4519  p->extras = ext1;
4520  p->last_chunk = last_chunk1;
4521  p->tc = tc1;
4522  p->num_t_min = num_t_min;
4523 #if OMPT_SUPPORT
4524  p->codeptr_ra = codeptr_ra;
4525 #endif
4526 
4527 #if OMPT_SUPPORT
4528  // schedule new task with correct return address for OMPT events
4529  __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra);
4530 #else
4531  __kmp_omp_task(gtid, new_task, true); // schedule new task
4532 #endif
4533 
4534  // execute the 1st half of current subrange
4535  if (n_tsk0 > num_t_min)
4536  __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0,
4537  ext0, last_chunk0, tc0, num_t_min,
4538 #if OMPT_SUPPORT
4539  codeptr_ra,
4540 #endif
4541  task_dup);
4542  else
4543  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
4544  gr_size0, ext0, last_chunk0, tc0,
4545 #if OMPT_SUPPORT
4546  codeptr_ra,
4547 #endif
4548  task_dup);
4549 
4550  KA_TRACE(40, ("__kmp_taskloop_recur(exit): T#%d\n", gtid));
4551 }
4552 
4553 static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
4554  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4555  int nogroup, int sched, kmp_uint64 grainsize,
4556  int modifier, void *task_dup) {
4557  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4558  KMP_DEBUG_ASSERT(task != NULL);
4559  if (nogroup == 0) {
4560 #if OMPT_SUPPORT && OMPT_OPTIONAL
4561  OMPT_STORE_RETURN_ADDRESS(gtid);
4562 #endif
4563  __kmpc_taskgroup(loc, gtid);
4564  }
4565 
4566  // =========================================================================
4567  // calculate loop parameters
4568  kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4569  kmp_uint64 tc;
4570  // compiler provides global bounds here
4571  kmp_uint64 lower = task_bounds.get_lb();
4572  kmp_uint64 upper = task_bounds.get_ub();
4573  kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag
4574  kmp_uint64 num_tasks = 0, extras = 0;
4575  kmp_int64 last_chunk =
4576  0; // reduce grainsize of last task by last_chunk in strict mode
4577  kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks;
4578  kmp_info_t *thread = __kmp_threads[gtid];
4579  kmp_taskdata_t *current_task = thread->th.th_current_task;
4580 
4581  KA_TRACE(20, ("__kmp_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
4582  "grain %llu(%d, %d), dup %p\n",
4583  gtid, taskdata, lower, upper, st, grainsize, sched, modifier,
4584  task_dup));
4585 
4586  // compute trip count
4587  if (st == 1) { // most common case
4588  tc = upper - lower + 1;
4589  } else if (st < 0) {
4590  tc = (lower - upper) / (-st) + 1;
4591  } else { // st > 0
4592  tc = (upper - lower) / st + 1;
4593  }
4594  if (tc == 0) {
4595  KA_TRACE(20, ("__kmp_taskloop(exit): T#%d zero-trip loop\n", gtid));
4596  // free the pattern task and exit
4597  __kmp_task_start(gtid, task, current_task);
4598  // do not execute anything for zero-trip loop
4599  __kmp_task_finish<false>(gtid, task, current_task);
4600  return;
4601  }
4602 
4603 #if OMPT_SUPPORT && OMPT_OPTIONAL
4604  ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
4605  ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
4606  if (ompt_enabled.ompt_callback_work) {
4607  ompt_callbacks.ompt_callback(ompt_callback_work)(
4608  ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
4609  &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
4610  }
4611 #endif
4612 
4613  if (num_tasks_min == 0)
4614  // TODO: can we choose better default heuristic?
4615  num_tasks_min =
4616  KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE);
4617 
4618  // compute num_tasks/grainsize based on the input provided
4619  switch (sched) {
4620  case 0: // no schedule clause specified, we can choose the default
4621  // let's try to schedule (team_size*10) tasks
4622  grainsize = thread->th.th_team_nproc * 10;
4623  KMP_FALLTHROUGH();
4624  case 2: // num_tasks provided
4625  if (grainsize > tc) {
4626  num_tasks = tc; // too big num_tasks requested, adjust values
4627  grainsize = 1;
4628  extras = 0;
4629  } else {
4630  num_tasks = grainsize;
4631  grainsize = tc / num_tasks;
4632  extras = tc % num_tasks;
4633  }
4634  break;
4635  case 1: // grainsize provided
4636  if (grainsize > tc) {
4637  num_tasks = 1;
4638  grainsize = tc; // too big grainsize requested, adjust values
4639  extras = 0;
4640  } else {
4641  if (modifier) {
4642  num_tasks = (tc + grainsize - 1) / grainsize;
4643  last_chunk = tc - (num_tasks * grainsize);
4644  extras = 0;
4645  } else {
4646  num_tasks = tc / grainsize;
4647  // adjust grainsize for balanced distribution of iterations
4648  grainsize = tc / num_tasks;
4649  extras = tc % num_tasks;
4650  }
4651  }
4652  break;
4653  default:
4654  KMP_ASSERT2(0, "unknown scheduling of taskloop");
4655  }
4656 
4657  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
4658  (last_chunk < 0 ? last_chunk : extras));
4659  KMP_DEBUG_ASSERT(num_tasks > extras);
4660  KMP_DEBUG_ASSERT(num_tasks > 0);
4661  // =========================================================================
4662 
4663  // check if clause value first
4664  // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native)
4665  if (if_val == 0) { // if(0) specified, mark task as serial
4666  taskdata->td_flags.task_serial = 1;
4667  taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
4668  // always start serial tasks linearly
4669  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4670  grainsize, extras, last_chunk, tc,
4671 #if OMPT_SUPPORT
4672  OMPT_GET_RETURN_ADDRESS(0),
4673 #endif
4674  task_dup);
4675  // !taskdata->td_flags.native => currently force linear spawning of tasks
4676  // for GOMP_taskloop
4677  } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) {
4678  KA_TRACE(20, ("__kmp_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
4679  "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
4680  gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
4681  last_chunk));
4682  __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4683  grainsize, extras, last_chunk, tc, num_tasks_min,
4684 #if OMPT_SUPPORT
4685  OMPT_GET_RETURN_ADDRESS(0),
4686 #endif
4687  task_dup);
4688  } else {
4689  KA_TRACE(20, ("__kmp_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
4690  "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
4691  gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
4692  last_chunk));
4693  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4694  grainsize, extras, last_chunk, tc,
4695 #if OMPT_SUPPORT
4696  OMPT_GET_RETURN_ADDRESS(0),
4697 #endif
4698  task_dup);
4699  }
4700 
4701 #if OMPT_SUPPORT && OMPT_OPTIONAL
4702  if (ompt_enabled.ompt_callback_work) {
4703  ompt_callbacks.ompt_callback(ompt_callback_work)(
4704  ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data),
4705  &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
4706  }
4707 #endif
4708 
4709  if (nogroup == 0) {
4710 #if OMPT_SUPPORT && OMPT_OPTIONAL
4711  OMPT_STORE_RETURN_ADDRESS(gtid);
4712 #endif
4713  __kmpc_end_taskgroup(loc, gtid);
4714  }
4715  KA_TRACE(20, ("__kmp_taskloop(exit): T#%d\n", gtid));
4716 }
4717 
4734 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
4735  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup,
4736  int sched, kmp_uint64 grainsize, void *task_dup) {
4737  __kmp_assert_valid_gtid(gtid);
4738  KA_TRACE(20, ("__kmpc_taskloop(enter): T#%d\n", gtid));
4739  __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
4740  0, task_dup);
4741  KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid));
4742 }
4743 
4761 void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
4762  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4763  int nogroup, int sched, kmp_uint64 grainsize,
4764  int modifier, void *task_dup) {
4765  __kmp_assert_valid_gtid(gtid);
4766  KA_TRACE(20, ("__kmpc_taskloop_5(enter): T#%d\n", gtid));
4767  __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
4768  modifier, task_dup);
4769  KA_TRACE(20, ("__kmpc_taskloop_5(exit): T#%d\n", gtid));
4770 }
struct kmp_taskred_data kmp_taskred_data_t
struct kmp_task_red_input kmp_task_red_input_t
struct kmp_taskred_flags kmp_taskred_flags_t
struct kmp_taskred_input kmp_taskred_input_t
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
Definition: kmp_stats.h:904
void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, void *task_dup)
void * __kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
void * __kmpc_taskred_init(int gtid, int num, void *data)
void * __kmpc_task_reduction_init(int gtid, int num, void *data)
void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask)
void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws)
kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 naffins, kmp_task_affinity_info_t *affin_list)
void * __kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, int modifier, void *task_dup)
void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask)
void * __kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data)
Definition: kmp.h:233
kmp_taskred_flags_t flags
kmp_taskred_flags_t flags
kmp_taskred_flags_t flags