Intel(R) Threading Building Blocks Doxygen Documentation  version 4.2.3
scheduler_common.h
Go to the documentation of this file.
1 /*
2  Copyright (c) 2005-2019 Intel Corporation
3 
4  Licensed under the Apache License, Version 2.0 (the "License");
5  you may not use this file except in compliance with the License.
6  You may obtain a copy of the License at
7 
8  http://www.apache.org/licenses/LICENSE-2.0
9 
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15 
16 
17 
18 
19 */
20 
21 #ifndef _TBB_scheduler_common_H
22 #define _TBB_scheduler_common_H
23 
24 #include "tbb/tbb_machine.h"
26 
27 #include <string.h> // for memset, memcpy, memmove
28 
29 #include "tbb_statistics.h"
30 
31 #if TBB_USE_ASSERT > 1
32 #include <stdio.h>
33 #endif /* TBB_USE_ASSERT > 1 */
34 
35 /* Temporarily change "private" to "public" while including "tbb/task.h".
36  This hack allows us to avoid publishing internal types and methods
37  in the public header files just for sake of friend declarations. */
38 #ifndef private
39  #define private public
40  #define undef_private
41 #endif
42 
43 #include "tbb/task.h"
44 #include "tbb/tbb_exception.h"
45 
46 #ifdef undef_private
47  #undef private
48 #endif
49 
50 #ifndef __TBB_SCHEDULER_MUTEX_TYPE
51 #define __TBB_SCHEDULER_MUTEX_TYPE tbb::spin_mutex
52 #endif
53 // TODO: add conditional inclusion based on specified type
54 #include "tbb/spin_mutex.h"
55 
56 // This macro is an attempt to get rid of ugly ifdefs in the shared parts of the code.
57 // It drops the second argument depending on whether the controlling macro is defined.
58 // The first argument is just a convenience allowing to keep comma before the macro usage.
59 #if __TBB_TASK_GROUP_CONTEXT
60  #define __TBB_CONTEXT_ARG1(context) context
61  #define __TBB_CONTEXT_ARG(arg1, context) arg1, context
62 #else /* !__TBB_TASK_GROUP_CONTEXT */
63  #define __TBB_CONTEXT_ARG1(context)
64  #define __TBB_CONTEXT_ARG(arg1, context) arg1
65 #endif /* !__TBB_TASK_GROUP_CONTEXT */
66 
67 #if __TBB_TASK_ISOLATION
68  #define __TBB_ISOLATION_EXPR(isolation) isolation
69  #define __TBB_ISOLATION_ARG(arg1, isolation) arg1, isolation
70 #else
71  #define __TBB_ISOLATION_EXPR(isolation)
72  #define __TBB_ISOLATION_ARG(arg1, isolation) arg1
73 #endif /* __TBB_TASK_ISOLATION */
74 
75 
76 #if DO_TBB_TRACE
77 #include <cstdio>
78 #define TBB_TRACE(x) ((void)std::printf x)
79 #else
80 #define TBB_TRACE(x) ((void)(0))
81 #endif /* DO_TBB_TRACE */
82 
83 #if !__TBB_CPU_CTL_ENV_PRESENT
84 #include <fenv.h>
85 #endif
86 
87 #if _MSC_VER && !defined(__INTEL_COMPILER)
88  // Workaround for overzealous compiler warnings
89  // These particular warnings are so ubiquitous that no attempt is made to narrow
90  // the scope of the warnings.
91  #pragma warning (disable: 4100 4127 4312 4244 4267 4706)
92 #endif
93 
94 namespace tbb {
95 namespace interface7 {
96 namespace internal {
97 class task_arena_base;
98 class delegated_task;
99 class wait_task;
100 }}
101 namespace internal {
102 using namespace interface7::internal;
103 
104 class arena;
105 template<typename SchedulerTraits> class custom_scheduler;
106 class generic_scheduler;
107 class governor;
108 class mail_outbox;
109 class market;
110 class observer_proxy;
111 class task_scheduler_observer_v3;
112 
113 #if __TBB_TASK_PRIORITY
114 static const intptr_t num_priority_levels = 3;
115 static const intptr_t normalized_normal_priority = (num_priority_levels - 1) / 2;
116 
117 inline intptr_t normalize_priority ( priority_t p ) {
118  return intptr_t(p - priority_low) / priority_stride_v4;
119 }
120 
121 static const priority_t priority_from_normalized_rep[num_priority_levels] = {
123 };
124 
125 inline void assert_priority_valid ( intptr_t p ) {
126  __TBB_ASSERT_EX( p >= 0 && p < num_priority_levels, NULL );
127 }
128 
129 inline intptr_t& priority ( task& t ) {
130  return t.prefix().context->my_priority;
131 }
132 #else /* __TBB_TASK_PRIORITY */
133 static const intptr_t num_priority_levels = 1;
134 #endif /* __TBB_TASK_PRIORITY */
135 
138 
139 #if __TBB_TASK_GROUP_CONTEXT
140 
150 extern uintptr_t the_context_state_propagation_epoch;
151 
153 
154 typedef scheduler_mutex_type context_state_propagation_mutex_type;
155 extern context_state_propagation_mutex_type the_context_state_propagation_mutex;
156 #endif /* __TBB_TASK_GROUP_CONTEXT */
157 
159 const size_t task_alignment = 32;
160 
162 
163 const size_t task_prefix_reservation_size = ((sizeof(internal::task_prefix)-1)/task_alignment+1)*task_alignment;
164 
171 #if __TBB_PREVIEW_CRITICAL_TASKS
172  es_task_critical = 0x8,
174 #endif
175  es_task_enqueued = 0x10,
183 };
184 
185 inline void reset_extra_state ( task *t ) {
186  t->prefix().extra_state &= ~(es_task_is_stolen | es_task_enqueued);
187 }
188 
196 
199 
202  no_cache = 4,
205 };
206 
207 //------------------------------------------------------------------------
208 // Debugging support
209 //------------------------------------------------------------------------
210 
211 #if TBB_USE_ASSERT
212 
214 
215 template <typename T>
216 void poison_value ( T& val ) { val = * punned_cast<T*>(&venom); }
217 
219 inline bool is_alive( uintptr_t v ) { return v != venom; }
220 
223 inline void assert_task_valid( const task* task ) {
224  __TBB_ASSERT( task!=NULL, NULL );
225  __TBB_ASSERT( !is_poisoned(&task), NULL );
226  __TBB_ASSERT( (uintptr_t)task % task_alignment == 0, "misaligned task" );
227 #if __TBB_RECYCLE_TO_ENQUEUE
228  __TBB_ASSERT( (unsigned)task->state()<=(unsigned)task::to_enqueue, "corrupt task (invalid state)" );
229 #else
230  __TBB_ASSERT( (unsigned)task->state()<=(unsigned)task::recycle, "corrupt task (invalid state)" );
231 #endif
232 }
233 
234 #else /* !TBB_USE_ASSERT */
235 
238 #define poison_value(g) ((void)0)
239 
240 inline void assert_task_valid( const task* ) {}
241 
242 #endif /* !TBB_USE_ASSERT */
243 
244 //------------------------------------------------------------------------
245 // Helpers
246 //------------------------------------------------------------------------
247 
248 #if __TBB_TASK_GROUP_CONTEXT
249 inline bool ConcurrentWaitsEnabled ( task& t ) {
250  return (t.prefix().context->my_version_and_traits & task_group_context::concurrent_wait) != 0;
251 }
252 
253 inline bool CancellationInfoPresent ( task& t ) {
254  return t.prefix().context->my_cancellation_requested != 0;
255 }
256 
257 #if TBB_USE_CAPTURED_EXCEPTION
258  inline tbb_exception* TbbCurrentException( task_group_context*, tbb_exception* src) { return src->move(); }
259  inline tbb_exception* TbbCurrentException( task_group_context* c, captured_exception* src) {
260  if( c->my_version_and_traits & task_group_context::exact_exception )
261  runtime_warning( "Exact exception propagation is requested by application but the linked library is built without support for it");
262  return src;
263  }
264  #define TbbRethrowException(TbbCapturedException) (TbbCapturedException)->throw_self()
265 #else
266  // Using macro instead of an inline function here allows to avoid evaluation of the
267  // TbbCapturedException expression when exact propagation is enabled for the context.
268  #define TbbCurrentException(context, TbbCapturedException) \
269  context->my_version_and_traits & task_group_context::exact_exception \
270  ? tbb_exception_ptr::allocate() \
271  : tbb_exception_ptr::allocate( *(TbbCapturedException) );
272  #define TbbRethrowException(TbbCapturedException) \
273  { \
274  if( governor::rethrow_exception_broken() ) fix_broken_rethrow(); \
275  (TbbCapturedException)->throw_self(); \
276  }
277 #endif /* !TBB_USE_CAPTURED_EXCEPTION */
278 
279 #define TbbRegisterCurrentException(context, TbbCapturedException) \
280  if ( context->cancel_group_execution() ) { \
281  /* We are the first to signal cancellation, so store the exception that caused it. */ \
282  context->my_exception = TbbCurrentException( context, TbbCapturedException ); \
283  }
284 
285 #define TbbCatchAll(context) \
286  catch ( tbb_exception& exc ) { \
287  TbbRegisterCurrentException( context, &exc ); \
288  } catch ( std::exception& exc ) { \
289  TbbRegisterCurrentException( context, captured_exception::allocate(typeid(exc).name(), exc.what()) ); \
290  } catch ( ... ) { \
291  TbbRegisterCurrentException( context, captured_exception::allocate("...", "Unidentified exception") );\
292  }
293 
294 #else /* !__TBB_TASK_GROUP_CONTEXT */
295 
296 inline bool ConcurrentWaitsEnabled ( task& t ) { return false; }
297 
298 #endif /* __TBB_TASK_GROUP_CONTEXT */
299 
300 inline void prolonged_pause() {
301 #if defined(__TBB_time_stamp) && !__TBB_STEALING_PAUSE
302  // Assumption based on practice: 1000-2000 ticks seems to be a suitable invariant for the
303  // majority of platforms. Currently, skip platforms that define __TBB_STEALING_PAUSE
304  // because these platforms require very careful tuning.
306  const machine_tsc_t finish = prev + 1000;
307  atomic_backoff backoff;
308  do {
309  backoff.bounded_pause();
311  if ( curr <= prev )
312  // Possibly, the current logical thread is moved to another hardware thread or overflow is occurred.
313  break;
314  prev = curr;
315  } while ( prev < finish );
316 #else
317 #ifdef __TBB_STEALING_PAUSE
318  static const long PauseTime = __TBB_STEALING_PAUSE;
319 #elif __TBB_ipf
320  static const long PauseTime = 1500;
321 #else
322  static const long PauseTime = 80;
323 #endif
324  // TODO IDEA: Update PauseTime adaptively?
325  __TBB_Pause(PauseTime);
326 #endif
327 }
328 
329 //------------------------------------------------------------------------
330 // arena_slot
331 //------------------------------------------------------------------------
333  //TODO: make this tbb:atomic<>.
335 
337 
338  // Synchronization of access to Task pool
343 
345 
347 };
348 
351 
352  unsigned hint_for_pop;
353 
354 #if __TBB_PREVIEW_CRITICAL_TASKS
355  unsigned hint_for_critical;
357 #endif
358 
360 
362 
365 
366  // Task pool of the scheduler that owns this slot
368 
369 #if __TBB_STATISTICS
370  statistics_counters *my_counters;
372 #endif /* __TBB_STATISTICS */
373 };
374 
375 struct arena_slot : padded<arena_slot_line1>, padded<arena_slot_line2> {
376 #if TBB_USE_ASSERT
377  void fill_with_canary_pattern ( size_t first, size_t last ) {
378  for ( size_t i = first; i < last; ++i )
379  poison_pointer(task_pool_ptr[i]);
380  }
381 #else
382  void fill_with_canary_pattern ( size_t, size_t ) {}
383 #endif /* TBB_USE_ASSERT */
384 
385  void allocate_task_pool( size_t n ) {
386  size_t byte_size = ((n * sizeof(task*) + NFS_MaxLineSize - 1) / NFS_MaxLineSize) * NFS_MaxLineSize;
387  my_task_pool_size = byte_size / sizeof(task*);
388  task_pool_ptr = (task**)NFS_Allocate( 1, byte_size, NULL );
389  // No need to clear the fresh deque since valid items are designated by the head and tail members.
390  // But fill it with a canary pattern in the high vigilance debug mode.
391  fill_with_canary_pattern( 0, my_task_pool_size );
392  }
393 
395  void free_task_pool( ) {
396  // TODO: understand the assertion and modify
397  // __TBB_ASSERT( !task_pool /*TODO: == EmptyTaskPool*/, NULL);
398  if( task_pool_ptr ) {
399  __TBB_ASSERT( my_task_pool_size, NULL);
400  NFS_Free( task_pool_ptr );
401  task_pool_ptr = NULL;
402  my_task_pool_size = 0;
403  }
404  }
405 };
406 
407 #if !__TBB_CPU_CTL_ENV_PRESENT
408 class cpu_ctl_env {
409  fenv_t *my_fenv_ptr;
410 public:
411  cpu_ctl_env() : my_fenv_ptr(NULL) {}
413  if ( my_fenv_ptr )
414  tbb::internal::NFS_Free( (void*)my_fenv_ptr );
415  }
416  // It is possible not to copy memory but just to copy pointers but the following issues should be addressed:
417  // 1. The arena lifetime and the context lifetime are independent;
418  // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside
419  // dispatch loop may become invalid.
420  // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation
421  // with a platform specific implementation.
422  cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(NULL) {
423  *this = src;
424  }
426  __TBB_ASSERT( src.my_fenv_ptr, NULL );
427  if ( !my_fenv_ptr )
428  my_fenv_ptr = (fenv_t*)tbb::internal::NFS_Allocate(1, sizeof(fenv_t), NULL);
429  *my_fenv_ptr = *src.my_fenv_ptr;
430  return *this;
431  }
432  bool operator!=( const cpu_ctl_env &ctl ) const {
433  __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." );
434  __TBB_ASSERT( ctl.my_fenv_ptr, "cpu_ctl_env is not initialized." );
435  return memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) );
436  }
437  void get_env () {
438  if ( !my_fenv_ptr )
439  my_fenv_ptr = (fenv_t*)tbb::internal::NFS_Allocate(1, sizeof(fenv_t), NULL);
440  fegetenv( my_fenv_ptr );
441  }
442  const cpu_ctl_env& set_env () const {
443  __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." );
444  fesetenv( my_fenv_ptr );
445  return *this;
446  }
447 };
448 #endif /* !__TBB_CPU_CTL_ENV_PRESENT */
449 
450 } // namespace internal
451 } // namespace tbb
452 
453 #endif /* _TBB_scheduler_common_H */
void __TBB_EXPORTED_FUNC runtime_warning(const char *format,...)
Report a runtime warning.
#define __TBB_time_stamp()
A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size...
Definition: tbb_stddef.h:477
void *__TBB_EXPORTED_FUNC NFS_Allocate(size_t n_element, size_t element_size, void *hint)
Allocate memory on cache/sector line boundary.
Tag for v3 tasks (i.e. tasks in TBB 2.1-2.2)
void fill_with_canary_pattern(size_t, size_t)
Set if ref_count might be changed by another thread. Used for debugging.
Disable caching for a small task.
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
Definition: tbb_stddef.h:169
Task is known to have been allocated by this scheduler.
const cpu_ctl_env & set_env() const
auto first(Container &c) -> decltype(begin(c))
Task is known to be a small task and must not be cached.
Task is known to be a small task.
bool bounded_pause()
Pause for a few times and return false if saturated.
Definition: tbb_machine.h:376
void allocate_task_pool(size_t n)
void reset_extra_state(task *t)
cpu_ctl_env & operator=(const cpu_ctl_env &src)
__TBB_atomic size_t head
Index of the first ready task in the deque.
Base class for user-defined tasks.
Definition: task.h:592
Work stealing task scheduler.
Definition: scheduler.h:124
Tag for v3 task_proxy.
Set if the task has been stolen.
#define __TBB_ASSERT_EX(predicate, comment)
"Extended" version is useful to suppress warnings if a variable is only used with an assert
Definition: tbb_stddef.h:171
#define __TBB_SCHEDULER_MUTEX_TYPE
generic_scheduler * my_scheduler
Scheduler of the thread attached to the slot.
void const char const char int ITT_FORMAT __itt_group_sync p
free_task_hint
Optimization hint to free_task that enables it omit unnecessary tests and code.
task_extra_state
Definitions for bits in task_prefix::extra_state.
#define poison_value(g)
auto last(Container &c) -> decltype(begin(c))
priority_t
Definition: task.h:295
uint64_t machine_tsc_t
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task * task
__TBB_atomic size_t tail
Index of the element following the last ready task in the deque.
static const intptr_t num_priority_levels
The graph class.
__TBB_SCHEDULER_MUTEX_TYPE scheduler_mutex_type
Mutex type for global locks in the scheduler.
bool operator!=(const cpu_ctl_env &ctl) const
static const int priority_stride_v4
Definition: task.h:288
task to be recycled as continuation
Definition: task.h:624
void poison_pointer(T *__TBB_atomic &)
Definition: tbb_stddef.h:309
Class that implements exponential backoff.
Definition: tbb_machine.h:349
const size_t task_alignment
Alignment for a task object.
#define __TBB_STEALING_PAUSE
Definition: mic_common.h:44
const size_t NFS_MaxLineSize
Compile-time constant that is upper bound on cache line/sector size.
Definition: tbb_stddef.h:220
#define __TBB_atomic
Definition: tbb_stddef.h:241
void __TBB_EXPORTED_FUNC NFS_Free(void *)
Free memory allocated by NFS_Allocate.
void free_task_pool()
Deallocate task pool that was allocated by means of allocate_task_pool.
size_t my_task_pool_size
Capacity of the primary task pool (number of elements - pointers to task).
internal::task_prefix & prefix(internal::version_tag *=NULL) const
Get reference to corresponding task_prefix.
Definition: task.h:941
void assert_task_valid(const task *)
const size_t task_prefix_reservation_size
Number of bytes reserved for a task prefix.
bool ConcurrentWaitsEnabled(task &t)
Tag for v1 tasks (i.e. tasks in TBB 1.0 and 2.0)
Bitwise-OR of local_task and small_task.
cpu_ctl_env(const cpu_ctl_env &src)
unsigned hint_for_pop
Hint provided for operations with the container of starvation-resistant tasks.
void __TBB_Pause(int32_t)
Definition: tbb_machine.h:335
Pads type T to fill out to a multiple of cache line size.
Definition: tbb_stddef.h:265

Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.