static inline void debug_work_deactivate(struct work_struct *work) { }
#endif
+#ifdef CONFIG_MTK_WQ_DEBUG
+extern void mttrace_workqueue_execute_work(struct work_struct *work);
+extern void mttrace_workqueue_activate_work(struct work_struct *work);
+extern void mttrace_workqueue_queue_work(unsigned int req_cpu, struct work_struct *work);
+extern void mttrace_workqueue_execute_end(struct work_struct *work);
+#endif //CONFIG_MTK_WQ_DEBUG
+
/* allocate ID and assign it to @pool */
static int worker_pool_assign_id(struct worker_pool *pool)
{
*/
smp_wmb();
set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
+ /*
+ * The following mb guarantees that previous clear of a PENDING bit
+ * will not be reordered with any speculative LOADS or STORES from
+ * work->current_func, which is executed afterwards. This possible
+ * reordering can lead to a missed execution on attempt to qeueue
+ * the same @work. E.g. consider this case:
+ *
+ * CPU#0 CPU#1
+ * ---------------------------- --------------------------------
+ *
+ * 1 STORE event_indicated
+ * 2 queue_work_on() {
+ * 3 test_and_set_bit(PENDING)
+ * 4 } set_..._and_clear_pending() {
+ * 5 set_work_data() # clear bit
+ * 6 smp_mb()
+ * 7 work->current_func() {
+ * 8 LOAD event_indicated
+ * }
+ *
+ * Without an explicit full barrier speculative LOAD on line 8 can
+ * be executed before CPU#0 does STORE on line 1. If that happens,
+ * CPU#0 observes the PENDING bit is still set and new execution of
+ * a @work is not queued in a hope, that CPU#1 will eventually
+ * finish the queued @work. Meanwhile CPU#1 does not see
+ * event_indicated is set, because speculative LOAD was executed
+ * before actual STORE.
+ */
+ smp_mb();
}
static void clear_work_data(struct work_struct *work)
struct pool_workqueue *pwq = get_work_pwq(work);
trace_workqueue_activate_work(work);
+#ifdef CONFIG_MTK_WQ_DEBUG
+ mttrace_workqueue_activate_work(work);
+#endif //CONFIG_MTK_WQ_DEBUG
move_linked_works(work, &pwq->pool->worklist, NULL);
__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
pwq->nr_active++;
/* pwq determined, queue */
trace_workqueue_queue_work(req_cpu, pwq, work);
+#ifdef CONFIG_MTK_WQ_DEBUG
+ mttrace_workqueue_queue_work(cpu, work);
+#endif //CONFIG_MTK_WQ_DEBUG
if (WARN_ON(!list_empty(&work->entry))) {
spin_unlock(&pwq->pool->lock);
if (likely(pwq->nr_active < pwq->max_active)) {
trace_workqueue_activate_work(work);
+#ifdef CONFIG_MTK_WQ_DEBUG
+ mttrace_workqueue_activate_work(work);
+#endif //CONFIG_MTK_WQ_DEBUG
pwq->nr_active++;
worklist = &pwq->pool->worklist;
} else {
bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
int work_color;
struct worker *collision;
+ unsigned long long exec_start;
+ char func[128];
+
#ifdef CONFIG_LOCKDEP
/*
* It is permissible to free the struct work_struct from
lock_map_acquire_read(&pwq->wq->lockdep_map);
lock_map_acquire(&lockdep_map);
+
+ exec_start = sched_clock();
+ sprintf(func, "%pf", work->func);
+
trace_workqueue_execute_start(work);
+#ifdef CONFIG_MTK_WQ_DEBUG
+ mttrace_workqueue_execute_work(work);
+#endif //CONFIG_MTK_WQ_DEBUG
+
worker->current_func(work);
+
/*
* While we must be careful to not use "work" after this, the trace
* point will only record its address.
*/
trace_workqueue_execute_end(work);
+#ifdef CONFIG_MTK_WQ_DEBUG
+ mttrace_workqueue_execute_end(work);
+#endif //CONFIG_MTK_WQ_DEBUG
+
+ if ((sched_clock() - exec_start)> 1000000000) // dump log if execute more than 1 sec
+ pr_warning("WQ warning! work (%s, %p) execute more than 1 sec, time: %llu ns\n", func, work, sched_clock() - exec_start);
+
lock_map_release(&lockdep_map);
lock_map_release(&pwq->wq->lockdep_map);
* attributes breaks ordering guarantee. Disallow exposing ordered
* workqueues.
*/
- if (WARN_ON(wq->flags & __WQ_ORDERED))
+ if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
return -EINVAL;
wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
return -EINVAL;
/* creating multiple pwqs breaks ordering guarantee */
- if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
- return -EINVAL;
+ if (!list_empty(&wq->pwqs)) {
+ if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
+ return -EINVAL;
+
+ wq->flags &= ~__WQ_ORDERED;
+ }
pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
struct workqueue_struct *wq;
struct pool_workqueue *pwq;
+ /*
+ * Unbound && max_active == 1 used to imply ordered, which is no
+ * longer the case on NUMA machines due to per-node pools. While
+ * alloc_ordered_workqueue() is the right way to create an ordered
+ * workqueue, keep the previous behavior to avoid subtle breakages
+ * on NUMA.
+ */
+ if ((flags & WQ_UNBOUND) && max_active == 1)
+ flags |= __WQ_ORDERED;
+
/* allocate wq and format name */
if (flags & WQ_UNBOUND)
tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
struct pool_workqueue *pwq;
/* disallow meddling with max_active for ordered workqueues */
- if (WARN_ON(wq->flags & __WQ_ORDERED))
+ if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
return;
max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
mutex_lock(&wq->mutex);
+ wq->flags &= ~__WQ_ORDERED;
wq->saved_max_active = max_active;
for_each_pwq(pwq, wq)
projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blobdiff