libutil++  1.9.3
 All Classes Functions Variables
ThreadPool.cpp
1 /*
2 ** libutil++
3 ** $Id: ThreadPool.cpp 1653 2016-02-28 19:54:59Z sella $
4 ** Copyright (c) 2011-2016 Digital Genesis, LLC. All Rights Reserved.
5 ** Released under the LGPL Version 2.1 License.
6 ** http://www.digitalgenesis.com
7 */
8 
9 static const char rcsid[] __attribute__((used)) = "$Id: ThreadPool.cpp 1653 2016-02-28 19:54:59Z sella $";
10 
11 #include "ThreadPool.h"
12 #include "CommonMacro.h"
13 #include "Time.h"
14 
15 #include <stdio.h>
16 #include <assert.h>
17 #include <stdlib.h>
18 #include <unistd.h>
19 #include <sys/time.h>
20 #include <syscall.h>
21 
22 #include <algorithm>
23 
24 using namespace sella::util;
25 using namespace std::chrono;
26 
27 ThreadPool::ThreadPool(size_t workers) throw (ThreadPoolException) :
28  TaskQueue(workers * 2),
29  threads(0),
30  creating(0),
31  actives(0),
32  overage(0),
33  shutdown(false),
34  paused(false),
35  allcpus(true),
36  maximum(workers),
37  minimum(workers),
38  spares(DefaultSpareWorkers)
39 {
40  clearCPUAffinity();
41  normalize();
42 
43  TRACE(PACKAGE, "ThreadPool starting up with %zd workers", this->maximum);
44 
45  try {
46  managerTID = std::thread(&ThreadPool::manager, this);
47  tids.insert(managerTID.native_handle());
48  } catch (std::system_error &e) {
49  THROW(ThreadPoolException, "Failed to start manager thread: %s", e.what());
50  }
51 }
52 
53 ThreadPool::ThreadPool(size_t maximumWorkers, size_t minimumWorkers, size_t spareWorkers) throw (ThreadPoolException) :
54  TaskQueue(maximumWorkers * 2),
55  threads(0),
56  creating(0),
57  actives(0),
58  overage(0),
59  shutdown(false),
60  paused(false),
61  allcpus(true),
62  maximum(maximumWorkers),
63  minimum(minimumWorkers),
64  spares(spareWorkers)
65 {
66  clearCPUAffinity();
67  normalize();
68 
69  TRACE(PACKAGE, "ThreadPool starting up with %zd/%zd workers, %zd spares", this->minimum, this->maximum, this->spares);
70 
71  try {
72  managerTID = std::thread(&ThreadPool::manager, this);
73  tids.insert(managerTID.native_handle());
74  } catch (std::system_error &e) {
75  THROW(ThreadPoolException, "Failed to start manager thread: %s", e.what());
76  }
77 }
78 
79 ThreadPool::ThreadPool(Log::shared log, size_t workers) throw (ThreadPoolException) :
80  TaskQueue(log, workers * 2),
81  threads(0),
82  creating(0),
83  actives(0),
84  overage(0),
85  shutdown(false),
86  paused(false),
87  allcpus(true),
88  maximum(workers),
89  minimum(workers),
90  spares(DefaultSpareWorkers)
91 {
92  clearCPUAffinity();
93  normalize();
94 
95  TRACE(PACKAGE, "ThreadPool starting up with %zd workers", this->maximum);
96 
97  try {
98  managerTID = std::thread(&ThreadPool::manager, this);
99  tids.insert(managerTID.native_handle());
100  } catch (std::system_error &e) {
101  THROW(ThreadPoolException, "Failed to start manager thread: %s", e.what());
102  }
103 }
104 
105 ThreadPool::ThreadPool(Log::shared log, size_t maximumWorkers, size_t minimumWorkers, size_t spareWorkers) throw (ThreadPoolException) :
106  TaskQueue(log, maximumWorkers * 2),
107  threads(0),
108  creating(0),
109  actives(0),
110  overage(0),
111  shutdown(false),
112  paused(false),
113  allcpus(true),
114  maximum(maximumWorkers),
115  minimum(minimumWorkers),
116  spares(spareWorkers)
117 {
118  clearCPUAffinity();
119  normalize();
120 
121  TRACE(PACKAGE, "ThreadPool starting up with %zd/%zd workers, %zd spares", this->minimum, this->maximum, this->spares);
122 
123  try {
124  managerTID = std::thread(&ThreadPool::manager, this);
125  tids.insert(managerTID.native_handle());
126  } catch (std::system_error &e) {
127  THROW(ThreadPoolException, "Failed to start manager thread: %s", e.what());
128  }
129 }
130 
131 ThreadPool::~ThreadPool() {
132  TRACE(PACKAGE, "ThreadPool shutting down with %zd queued, %zd workers, %zd active", heap.size(), (size_t) threads, (size_t) actives);
133 
134  shutdown = true;
135  managerMutex.lock();
136  managerCV.notify_all();
137  managerMutex.unlock();
138 
139  if (managerTID.joinable()) {
140  managerTID.join();
141  }
142 
143  std::lock_guard<std::mutex> lg(emptyMutex);
144  emptyCV.notify_all();
145 }
146 
147 void ThreadPool::schedule(Task::shared &task, int priority) {
148  assert(task.get() != NULL);
149 
150  if (priority) task->setPriority(priority);
151 
152  TRACE(PACKAGE "-detail", "Scheduling task %p to run immedately with priority %d", task.get(), task->getPriority());
153 
154  heapMutex.lock();
155  push(task);
156  heapMutex.unlock();
157 
158  std::lock_guard<std::mutex> lg(managerMutex);
159  managerCV.notify_all();
160 }
161 
162 void ThreadPool::schedule(Task::shared &task, const std::chrono::system_clock::duration &delay, int priority) {
163  assert(task.get() != NULL);
164 
165  task->setDelay(delay);
166  if (priority) task->setPriority(priority);
167 
168  TRACE(PACKAGE "-detail", "Scheduling task %p to run in %lld.%lld sec with priority %d", task.get(), task->getDelay().tv_sec, task->getDelay().tv_usec, task->getPriority());
169 
170  heapMutex.lock();
171  push(task);
172  heapMutex.unlock();
173 
174  std::lock_guard<std::mutex> lg(managerMutex);
175  managerCV.notify_all();
176 }
177 
178 void ThreadPool::schedule(Task::shared &task, const std::chrono::system_clock::time_point &time, int priority) {
179  assert(task.get() != NULL);
180 
181  task->setTimer(time);
182  if (priority) task->setPriority(priority);
183 
184  TRACE(PACKAGE "-detail", "Scheduling task %p to run at %lld.%lld sec with priority %d", task.get(), task->getTimer().tv_sec, task->getTimer().tv_usec, task->getPriority());
185 
186  heapMutex.lock();
187  push(task);
188  heapMutex.unlock();
189 
190  std::lock_guard<std::mutex> lg(managerMutex);
191  managerCV.notify_all();
192 }
193 
194 void ThreadPool::schedule(Task::shared &task, time_t sec, suseconds_t usec, int priority) {
195  assert(task.get() != NULL);
196 
197  task->setDelay(sec, usec);
198  if (priority) task->setPriority(priority);
199 
200  TRACE(PACKAGE "-detail", "Scheduling task %p to run in %lld.%lld sec with priority %d", task.get(), sec, usec, task->getPriority());
201 
202  heapMutex.lock();
203  push(task);
204  heapMutex.unlock();
205 
206  std::lock_guard<std::mutex> lg(managerMutex);
207  managerCV.notify_all();
208 }
209 
210 void ThreadPool::schedule(Task::shared &task, timeval &tv, int priority) {
211  assert(task.get() != NULL);
212 
213  task->setTimer(tv);
214  if (priority) task->setPriority(priority);
215 
216  TRACE(PACKAGE "-detail", "Scheduling task %p to run at %lld.%lld sec with priority %d", task.get(), Time::tvtosec(tv), Time::tvtousec(tv), task->getPriority());
217 
218  heapMutex.lock();
219  push(task);
220  heapMutex.unlock();
221 
222  std::lock_guard<std::mutex> lg(managerMutex);
223  managerCV.notify_all();
224 }
225 
226 bool ThreadPool::reschedule(Task::shared &task, const std::chrono::system_clock::duration &delay, int priority, bool strict) {
227  assert(task.get() != NULL);
228 
229  task->setDelay(delay);
230  if (priority) task->setPriority(priority);
231 
232  TRACE(PACKAGE "-detail", "Rescheduling task %p to run in %lld.%lld sec with priority %d", task.get(), task->getDelay().tv_sec, task->getDelay().tv_usec, task->getPriority());
233 
234  heapMutex.lock();
235  bool res = repush(task, delay, priority, strict);
236  heapMutex.unlock();
237 
238  std::lock_guard<std::mutex> lg(managerMutex);
239  managerCV.notify_all();
240 
241  return res;
242 }
243 
244 bool ThreadPool::reschedule(Task::shared &task, const std::chrono::system_clock::time_point &time, int priority, bool strict) {
245  assert(task.get() != NULL);
246 
247  task->setTimer(time);
248  if (priority) task->setPriority(priority);
249 
250  TRACE(PACKAGE "-detail", "Rescheduling task %p to run at %lld.%lld sec with priority %d", task.get(), task->getTimer().tv_sec, task->getTimer().tv_usec, task->getPriority());
251 
252  heapMutex.lock();
253  bool res = repush(task, time, priority, strict);
254  heapMutex.unlock();
255 
256  std::lock_guard<std::mutex> lg(managerMutex);
257  managerCV.notify_all();
258 
259  return res;
260 }
261 
262 bool ThreadPool::reschedule(Task::shared &task, time_t sec, suseconds_t usec, int priority, bool strict) {
263  assert(task.get() != NULL);
264 
265  if (priority) task->setPriority(priority);
266 
267  TRACE(PACKAGE "-detail", "Rescheduling task %p to run in %lld.%lld sec with priority %d", task.get(), sec, usec, task->getPriority());
268 
269  heapMutex.lock();
270  bool res = repush(task, sec, usec, priority, strict);
271  heapMutex.unlock();
272 
273  std::lock_guard<std::mutex> lg(managerMutex);
274  managerCV.notify_all();
275 
276  return res;
277 }
278 
279 bool ThreadPool::reschedule(Task::shared &task, timeval &tv, int priority, bool strict) {
280  assert(task.get() != NULL);
281 
282  if (priority) task->setPriority(priority);
283 
284  TRACE(PACKAGE "-detail", "Rescheduling task %p to run at %lld.%lld sec with priority %d", task.get(), tv.tv_sec, tv.tv_usec, task->getPriority());
285 
286  heapMutex.lock();
287  bool res = repush(task, tv, priority, strict);
288  heapMutex.unlock();
289 
290  std::lock_guard<std::mutex> lg(managerMutex);
291  managerCV.notify_all();
292 
293  return res;
294 }
295 
296 bool ThreadPool::unschedule(Task::shared &task, bool firstonly) {
297  TRACE(PACKAGE "-detail", "Unscheduling task %p%s", task.get(), (firstonly) ? " (firstonly)" : "");
298 
299  heapMutex.lock();
300  bool res = erase(task, firstonly);
301  heapMutex.unlock();
302 
303  std::lock_guard<std::mutex> lg(managerMutex);
304  managerCV.notify_all();
305 
306  return res;
307 }
308 
309 void ThreadPool::pause(bool on) {
310  std::lock_guard<std::mutex> lg(managerMutex);
311 
312  if ((paused = on) == false) {
313  TRACE(PACKAGE, "Resuming %zd queued, %zd workers, %zd active", heap.size(), (size_t) threads, (size_t) actives);
314 
315  managerCV.notify_all();
316  } else {
317  TRACE(PACKAGE, "Pausing %zd queued, %zd workers, %zd active", heap.size(), (size_t) threads, (size_t) actives);
318  }
319 }
320 
321 size_t ThreadPool::getCPUCount(void) {
322  return (std::thread::hardware_concurrency() > 0) ? std::thread::hardware_concurrency() : sysconf(_SC_NPROCESSORS_ONLN);
323 }
324 
325 int ThreadPool::getCPUSetCount(void) {
326  return CPU_COUNT(&cpuset);
327 }
328 
329 bool ThreadPool::addCPUAffinity(std::set<int> cpus) {
330  size_t count = getCPUCount();
331 
332  if (allcpus) {
333  CPU_ZERO(&cpuset);
334  }
335 
336  for (auto c = cpus.begin(); c != cpus.end(); ++c) {
337  if ((size_t) *c < count) {
338  CPU_SET(*c, &cpuset);
339  }
340  }
341 
342  if (CPU_COUNT(&cpuset) == 0) {
343  clearCPUAffinity();
344  } else {
345  allcpus = false;
346  }
347 
348  return setThreadsAffinity();
349 }
350 
351 bool ThreadPool::addCPUAffinity(int cpu) {
352  size_t count = getCPUCount();
353 
354  if (allcpus) {
355  CPU_ZERO(&cpuset);
356  }
357 
358  if ((size_t) cpu < count) {
359  CPU_SET(cpu, &cpuset);
360  }
361 
362  if (CPU_COUNT(&cpuset) == 0) {
363  clearCPUAffinity();
364  } else {
365  allcpus = false;
366  }
367 
368  return setThreadsAffinity();
369 }
370 
371 bool ThreadPool::clearCPUAffinity(void) {
372  CPU_ZERO(&cpuset);
373  allcpus = true;
374 
375  size_t cpus = getCPUCount();
376 
377  for (size_t i = 0; i < cpus; i++) {
378  CPU_SET(i, &cpuset);
379  }
380 
381  return setThreadsAffinity();
382 }
383 
384 bool ThreadPool::clearCPUAffinity(int cpu) {
385  if (cpu < CPU_SETSIZE) {
386  CPU_CLR(cpu, &cpuset);
387  }
388 
389  if (CPU_COUNT(&cpuset) == 0) {
390  clearCPUAffinity();
391  }
392 
393  return setThreadsAffinity();
394 }
395 
396 bool ThreadPool::clearCPUAffinity(std::set<int> cpus) {
397  for (auto c = cpus.begin(); c != cpus.end(); ++c) {
398  if (*c < CPU_SETSIZE) {
399  CPU_CLR(*c, &cpuset);
400  }
401  }
402 
403  if (CPU_COUNT(&cpuset) == 0) {
404  clearCPUAffinity();
405  }
406 
407  return setThreadsAffinity();
408 }
409 
410 size_t ThreadPool::size(void) const {
411  std::lock_guard<std::mutex> lg(heapMutex);
412 
413  return TaskQueue::size() + actives;
414 }
415 
416 size_t ThreadPool::workers(void) const {
417  return threads;
418 }
419 
420 size_t ThreadPool::active(void) const {
421  return actives;
422 }
423 
424 bool ThreadPool::empty(void) const {
425  return (size() == 0);
426 }
427 
428 void ThreadPool::clear(void) {
429  heapMutex.lock();
430  TaskQueue::clear();
431  heapMutex.unlock();
432 
433  std::lock_guard<std::mutex> lg(managerMutex);
434  managerCV.notify_all();
435 }
436 
437 Task::shared ThreadPool::next(void) {
438  std::lock_guard<std::mutex> lg(heapMutex);
439 
440  if (!TaskQueue::empty()) {
441  Task::shared task = top();
442 
443  if (task && Task::compareTimer(task)) {
444  pop();
445 
446  return task;
447  }
448  }
449 
450  return Task::shared();
451 }
452 
453 microseconds ThreadPool::duration(void) const {
454  if (actives == maximum || UNLIKELY(paused && !shutdown)) {
455  return max();
456  }
457 
458  std::lock_guard<std::mutex> lg(heapMutex);
459 
460  if (!TaskQueue::empty()) {
461  Task::shared task = top();
462  timeval delay = task->getDelay();
463 
464  if (delay.tv_usec < 1 && delay.tv_sec < 1) {
465  if (creating) {
466  return milliseconds(10);
467  } else {
468  return milliseconds(2);
469  }
470  }
471 
472  return seconds(delay.tv_sec) + microseconds(delay.tv_usec);
473  } else if (UNLIKELY(shutdown)) {
474  return milliseconds(1);
475  }
476 
477  return max();
478 }
479 
480 microseconds ThreadPool::max(void) const {
481  return hours(24 * 365 * 100); // 100 years because cv.wait_for(microseconds::max()) broken on GCC 4.4.4.
482 }
483 
484 bool ThreadPool::ready(void) const {
485  if (TaskQueue::empty()) {
486  return false;
487  }
488 
489  std::lock_guard<std::mutex> lg(heapMutex);
490 
491  return Task::compareTimer(top());
492 }
493 
494 bool ThreadPool::wait(void) {
495  std::unique_lock<std::mutex> lock(emptyMutex);
496 
497  if (empty()) {
498  managerMutex.lock();
499  managerCV.notify_all();
500  managerMutex.unlock();
501 
502  return true;
503  }
504 
505  managerMutex.lock();
506  managerCV.notify_all();
507  managerMutex.unlock();
508 
509  emptyCV.wait(lock);
510 
511  return empty();
512 }
513 
514 bool ThreadPool::wait(const std::chrono::system_clock::duration &delay) {
515  std::unique_lock<std::mutex> lock(emptyMutex);
516 
517  if (empty()) {
518  managerMutex.lock();
519  managerCV.notify_all();
520  managerMutex.unlock();
521 
522  return true;
523  }
524 
525  managerMutex.lock();
526  managerCV.notify_all();
527  managerMutex.unlock();
528 
529  emptyCV.wait_for(lock, delay);
530 
531  return empty();
532 }
533 
534 bool ThreadPool::wait(const std::chrono::system_clock::time_point &time) {
535  std::unique_lock<std::mutex> lock(emptyMutex);
536 
537  if (empty()) {
538  managerMutex.lock();
539  managerCV.notify_all();
540  managerMutex.unlock();
541 
542  return true;
543  }
544 
545  managerMutex.lock();
546  managerCV.notify_all();
547  managerMutex.unlock();
548 
549  emptyCV.wait_until(lock, time);
550 
551  return empty();
552 }
553 
554 void ThreadPool::reserve(size_t capacity) {
555  std::lock_guard<std::mutex> lg(heapMutex);
556  TaskQueue::reserve(capacity);
557 }
558 
559 void ThreadPool::print(void) const {
560  std::lock_guard<std::mutex> lg(heapMutex);
561 
562  Task::shared task;
563  timeval now, timer, delay;
564 
565  if (libutilxx__log == NULL) {
566  return;
567  }
568 
569  gettimeofday(&now, NULL);
570  INFO("[ThreadPool Heap] - now %10lu.%06lu, %zd, queued, %zd workers, %zd active, %zd overage", now.tv_sec, now.tv_usec, heap.size(), (size_t) threads, (size_t) actives, (size_t) overage);
571 
572  if (heap.empty()) {
573  INFO(" heap => empty");
574 
575  return;
576  }
577 
578  for (auto it = heap.begin(); it != heap.end(); ++it) {
579  timer = (*it)->getTimer();
580  if (timerisset(&timer)) {
581  /* The output comes out very wrong when timersub() is used. This output is odd, but accurate. */
582  delay.tv_sec = timer.tv_sec - now.tv_sec;
583  delay.tv_usec = timer.tv_usec - now.tv_usec;
584  if (delay.tv_sec > 0 && delay.tv_usec < 0) { /* Only preform the subtraction when tv_sec is > 0. */
585  --delay.tv_sec;
586  delay.tv_usec += 1000000;
587  }
588 
589  INFO(" heap => task(%10p) obj: %10p pri: %3d timer: %10lu.%06lu delay: %4d.%d", (*it).get(), (*it)->getObject(), (*it)->getPriority(), timer.tv_sec, timer.tv_usec, (signed) delay.tv_sec, (signed) delay.tv_usec);
590  } else {
591  INFO(" heap => task(%10p) obj: %10p pri: %3d timer: %10lu.%06lu delay: -inf", (*it).get(), (*it)->getObject(), (*it)->getPriority(), timer.tv_sec, timer.tv_usec);
592  }
593  }
594 }
595 
596 ThreadPool::shared ThreadPool::shared_ptr(size_t maximumWorkers) throw (ThreadPoolException) {
597  return std::make_shared<ThreadPool>(maximumWorkers);
598 }
599 
600 ThreadPool::shared ThreadPool::shared_ptr(size_t maximumWorkers, size_t minimumWorkers, size_t spareWorkers) throw (ThreadPoolException) {
601  return std::make_shared<ThreadPool>(maximumWorkers, minimumWorkers, spareWorkers);
602 }
603 
604 ThreadPool::shared ThreadPool::shared_ptr(Log::shared log, size_t maximumWorkers) throw (ThreadPoolException) {
605  return std::make_shared<ThreadPool>(log, maximumWorkers);
606 }
607 
608 ThreadPool::shared ThreadPool::shared_ptr(Log::shared log, size_t maximumWorkers, size_t minimumWorkers, size_t spareWorkers) throw (ThreadPoolException) {
609  return std::make_shared<ThreadPool>(log, maximumWorkers, minimumWorkers, spareWorkers);
610 }
611 
612 void ThreadPool::manager(void) {
613  size_t cycles = 0;
614 
615  for (size_t i = 0; !shutdown && i < minimum; i++) {
616  createWorker();
617  }
618 
619  TRACE(PACKAGE "-detail", "Manager started up with %zd threads", (size_t) threads);
620 
621  while (LIKELY(!shutdown)) {
622  if (minimum == maximum) { /* Non-scaling manager */
623  while (LIKELY(!shutdown) && threads < minimum) {
624  createWorker();
625  }
626  } else { /* Scaling manager */
627  while (LIKELY(!shutdown) && threads < maximum && threads < actives + spares) {
628  createWorker();
629  }
630 
631  if (ready() || (++cycles % maximum) != 0) { /* Don't give up workers when ready tasks exist, or too few cycles have passed. */
632  overage = 0;
633  } else {
634  if (threads > minimum && threads > actives + spares) {
635  overage = threads - actives + spares;
636  } else if (actives == 0) { /* Drop back to minimum with no active tasks */
637  overage = threads - minimum;
638  }
639 
640  TRACE(PACKAGE "-detail", "Manager set overage to %zd", (size_t) overage);
641 
642  for (size_t i = 0; LIKELY(!shutdown) && i < overage && i < maximum; i++) { /* i < maximum handles underrun which may occur during shutdown */
643  TRACE(PACKAGE "-detail", "Manager waking worker for overage");
644 
645  std::lock_guard<std::mutex> lg(workerMutex);
646  workerCV.notify_one(); /* Wake required workers */
647  }
648  }
649  }
650 
651  if (LIKELY(!shutdown)) {
652  TRACE(PACKAGE, "Manager waiting for tasks - %zd queued, %zd active, %zd/%zd workers", heap.size(), (size_t) actives, (size_t) threads, maximum);
653 
654  std::unique_lock<std::mutex> lock(managerMutex);
655  do {
656  emptyMutex.lock();
657  if (empty()) {
658  emptyCV.notify_all();
659  }
660  emptyMutex.unlock();
661 
662  microseconds delay = duration();
663 #if 0
664  TRACE(PACKAGE "-detail", "delay: %lld, active: %zd, threads: %zd, overage: %zd, shutdown: %d", microseconds(delay).count(), (size_t) actives, (size_t) threads, (size_t) overage, shutdown);
665 #endif
666  managerCV.wait_for(lock, delay);
667  } while (UNLIKELY(paused && !shutdown));
668  }
669 
670  if (LIKELY(!shutdown)) {
671  size_t max = maximum - actives;
672 
673  TRACE(PACKAGE, "Manager waking %zd worker(s) - %zd queued, %zd active, %zd/%zd workers", max, heap.size(), (size_t) actives, (size_t) threads, maximum);
674 
675  std::lock_guard<std::mutex> lg(workerMutex);
676  for (size_t i = 0; ready() && i < max; i++) { /* Loop boosts performance for quick tasks. */
677  workerCV.notify_one();
678  }
679  }
680  }
681 
682  TRACE(PACKAGE "-detail", "Manager waiting on remaining %zd workers to complete", (size_t) threads);
683 
684  clear();
685  overage = (size_t) threads;
686  workerCV.notify_all();
687 
688  while (threads > 0) { /* Loop until all threads are complete */
689  std::unique_lock<std::mutex> lock(managerMutex);
690  managerCV.wait_for(lock, seconds(1));
691 
692  workerCV.notify_all();
693  }
694 
695  std::lock_guard<std::mutex> lg(emptyMutex);
696  emptyCV.notify_all();
697 
698  TRACE(PACKAGE, "Manager shutdown");
699 }
700 
701 void ThreadPool::worker(void) {
702  Task::shared task;
703  int tid = syscall(SYS_gettid);
704 
705  TRACE(PACKAGE, "Worker(%d) started", tid);
706 
707  actives++;
708  creating--;
709 
710  while (LIKELY(!shutdown)) {
711  {
712  TRACE(PACKAGE "-detail", "Worker(%d) waiting for next task", tid);
713 
714  std::unique_lock<std::mutex> lock(workerMutex);
715 
716  managerMutex.lock();
717  managerCV.notify_all();
718  managerMutex.unlock();
719 
720  --actives;
721  workerCV.wait(lock);
722  ++actives;
723  }
724 
725  if (LIKELY(!shutdown)) {
726  if (overage) { /* Check if this thread is unneeded */
727  std::lock_guard<std::mutex> lg(overageMutex);
728 
729  if (overage) {
730  TRACE(PACKAGE "-detail", "Worker(%d) yielding up the ghost (spare worker)", tid);
731 
732  --overage;
733 
734  break;
735  }
736  }
737 
738  workerMutex.lock();
739 
740  if (ready()) {
741  task = next();
742 
743  TRACE(PACKAGE "-detail", "Worker(%d) acquired task %p with priority %d", tid, task.get(), task->getPriority());
744  }
745 
746  workerMutex.unlock();
747 
748  if (task) {
749  TRACE(PACKAGE, "Worker(%d) executing task %p with priority %d", tid, task.get(), task->getPriority());
750 
751  task->execute();
752  task.reset();
753  }
754  }
755  }
756 
757  TRACE(PACKAGE, "Worker(%d) ended", tid);
758 
759  tidsMutex.lock();
760  tids.erase(pthread_self());
761  tidsMutex.unlock();
762 
763  --actives;
764  --threads;
765 
766  std::lock_guard<std::mutex> lg(managerMutex);
767  managerCV.notify_all();
768 }
769 
770 void ThreadPool::normalize(void) {
771  if (maximum == 0) {
772  maximum = getCPUCount();
773  }
774 
775  if (minimum == 0) {
776  minimum = getCPUCount();
777  }
778 
779  if (spares == 0) {
780  spares = DefaultSpareWorkers;
781  }
782 
783  if (minimum > maximum) {
784  maximum = minimum;
785  spares = 0;
786  } else if (maximum == minimum) {
787  spares = 0;
788  }
789 }
790 
791 void ThreadPool::createWorker(void) {
792  TRACE(PACKAGE "-detail", "Manager starting up worker %zd/%zd with affinity to %d CPUs", (size_t) threads + 1, maximum, getCPUSetCount());
793 
794  try {
795  std::thread t(&ThreadPool::worker, this);
796  setThreadAffinity(t.native_handle());
797 
798  tidsMutex.lock();
799  tids.insert(t.native_handle());
800  tidsMutex.unlock();
801 
802  ++threads;
803  ++creating;
804  t.detach();
805  } catch (std::exception &e) {
806  TRACE(PACKAGE, "Failed to create worker thread: %s", e.what());
807  }
808 }
809 
810 bool ThreadPool::setThreadsAffinity() {
811  std::lock_guard<std::mutex> lg(tidsMutex);
812 
813  TRACE(PACKAGE "-detail", "Setting %zd threads affinity to %d CPUs", tids.size(), getCPUSetCount());
814 
815  for (auto t = tids.begin(); t != tids.end(); ++t) {
816  if (!setThreadAffinity(*t)) {
817  return false;
818  }
819  }
820 
821  return true;
822 }
823 
824 bool ThreadPool::setThreadAffinity(pthread_t tid) {
825  int error;
826 
827  if ((error = pthread_setaffinity_np(tid, sizeof(cpu_set_t), &cpuset)) != 0) {
828  char buf[1024] = "message unavailable";
829  char *e = strerror_r(error, buf, sizeof(buf));
830  (void) e; // Prevent warning
831 
832  TRACE(PACKAGE, "pthread_setaffinity_np: %d:%s", error, buf);
833 
834  return false;
835  }
836 
837  return true;
838 }
839 
840 /*
841 ** vim: noet ts=3 sw=3
842 */