1 // SPDX-License-Identifier: GPL-2.0
3 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
9 #include <linux/slab.h>
10 #include <linux/irq_work.h>
14 #include <trace/events/sched.h>
16 int sched_rr_timeslice
= RR_TIMESLICE
;
17 int sysctl_sched_rr_timeslice
= (MSEC_PER_SEC
/ HZ
) * RR_TIMESLICE
;
20 void update_rt_load_avg(u64 now
, struct sched_rt_entity
*rt_se
);
22 static int do_sched_rt_period_timer(struct rt_bandwidth
*rt_b
, int overrun
);
24 struct rt_bandwidth def_rt_bandwidth
;
26 static enum hrtimer_restart
sched_rt_period_timer(struct hrtimer
*timer
)
28 struct rt_bandwidth
*rt_b
=
29 container_of(timer
, struct rt_bandwidth
, rt_period_timer
);
33 raw_spin_lock(&rt_b
->rt_runtime_lock
);
35 overrun
= hrtimer_forward_now(timer
, rt_b
->rt_period
);
39 raw_spin_unlock(&rt_b
->rt_runtime_lock
);
40 idle
= do_sched_rt_period_timer(rt_b
, overrun
);
41 raw_spin_lock(&rt_b
->rt_runtime_lock
);
44 rt_b
->rt_period_active
= 0;
45 raw_spin_unlock(&rt_b
->rt_runtime_lock
);
47 return idle
? HRTIMER_NORESTART
: HRTIMER_RESTART
;
50 void init_rt_bandwidth(struct rt_bandwidth
*rt_b
, u64 period
, u64 runtime
)
52 rt_b
->rt_period
= ns_to_ktime(period
);
53 rt_b
->rt_runtime
= runtime
;
55 raw_spin_lock_init(&rt_b
->rt_runtime_lock
);
57 hrtimer_init(&rt_b
->rt_period_timer
,
58 CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
59 rt_b
->rt_period_timer
.function
= sched_rt_period_timer
;
62 static void start_rt_bandwidth(struct rt_bandwidth
*rt_b
)
64 if (!rt_bandwidth_enabled() || rt_b
->rt_runtime
== RUNTIME_INF
)
67 raw_spin_lock(&rt_b
->rt_runtime_lock
);
68 if (!rt_b
->rt_period_active
) {
69 rt_b
->rt_period_active
= 1;
71 * SCHED_DEADLINE updates the bandwidth, as a run away
72 * RT task with a DL task could hog a CPU. But DL does
73 * not reset the period. If a deadline task was running
74 * without an RT task running, it can cause RT tasks to
75 * throttle when they start up. Kick the timer right away
76 * to update the period.
78 hrtimer_forward_now(&rt_b
->rt_period_timer
, ns_to_ktime(0));
79 hrtimer_start_expires(&rt_b
->rt_period_timer
, HRTIMER_MODE_ABS_PINNED
);
81 raw_spin_unlock(&rt_b
->rt_runtime_lock
);
84 void init_rt_rq(struct rt_rq
*rt_rq
)
86 struct rt_prio_array
*array
;
89 array
= &rt_rq
->active
;
90 for (i
= 0; i
< MAX_RT_PRIO
; i
++) {
91 INIT_LIST_HEAD(array
->queue
+ i
);
92 __clear_bit(i
, array
->bitmap
);
94 /* delimiter for bitsearch: */
95 __set_bit(MAX_RT_PRIO
, array
->bitmap
);
97 #if defined CONFIG_SMP
98 rt_rq
->highest_prio
.curr
= MAX_RT_PRIO
;
99 rt_rq
->highest_prio
.next
= MAX_RT_PRIO
;
100 rt_rq
->rt_nr_migratory
= 0;
101 rt_rq
->overloaded
= 0;
102 plist_head_init(&rt_rq
->pushable_tasks
);
103 atomic_long_set(&rt_rq
->removed_util_avg
, 0);
104 atomic_long_set(&rt_rq
->removed_load_avg
, 0);
105 #endif /* CONFIG_SMP */
106 /* We start is dequeued state, because no RT tasks are queued */
107 rt_rq
->rt_queued
= 0;
110 rt_rq
->rt_throttled
= 0;
111 rt_rq
->rt_runtime
= 0;
112 raw_spin_lock_init(&rt_rq
->rt_runtime_lock
);
115 #ifdef CONFIG_RT_GROUP_SCHED
116 static void destroy_rt_bandwidth(struct rt_bandwidth
*rt_b
)
118 hrtimer_cancel(&rt_b
->rt_period_timer
);
121 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
123 static inline struct task_struct
*rt_task_of(struct sched_rt_entity
*rt_se
)
125 #ifdef CONFIG_SCHED_DEBUG
126 WARN_ON_ONCE(!rt_entity_is_task(rt_se
));
128 return container_of(rt_se
, struct task_struct
, rt
);
131 static inline struct rq
*rq_of_rt_rq(struct rt_rq
*rt_rq
)
136 static inline struct rt_rq
*rt_rq_of_se(struct sched_rt_entity
*rt_se
)
141 static inline struct rq
*rq_of_rt_se(struct sched_rt_entity
*rt_se
)
143 struct rt_rq
*rt_rq
= rt_se
->rt_rq
;
148 void free_rt_sched_group(struct task_group
*tg
)
153 destroy_rt_bandwidth(&tg
->rt_bandwidth
);
155 for_each_possible_cpu(i
) {
166 void init_tg_rt_entry(struct task_group
*tg
, struct rt_rq
*rt_rq
,
167 struct sched_rt_entity
*rt_se
, int cpu
,
168 struct sched_rt_entity
*parent
)
170 struct rq
*rq
= cpu_rq(cpu
);
172 rt_rq
->highest_prio
.curr
= MAX_RT_PRIO
;
173 rt_rq
->rt_nr_boosted
= 0;
177 tg
->rt_rq
[cpu
] = rt_rq
;
178 tg
->rt_se
[cpu
] = rt_se
;
184 rt_se
->rt_rq
= &rq
->rt
;
186 rt_se
->rt_rq
= parent
->my_q
;
189 rt_se
->parent
= parent
;
190 INIT_LIST_HEAD(&rt_se
->run_list
);
193 int alloc_rt_sched_group(struct task_group
*tg
, struct task_group
*parent
)
196 struct sched_rt_entity
*rt_se
;
199 tg
->rt_rq
= kzalloc(sizeof(rt_rq
) * nr_cpu_ids
, GFP_KERNEL
);
202 tg
->rt_se
= kzalloc(sizeof(rt_se
) * nr_cpu_ids
, GFP_KERNEL
);
206 init_rt_bandwidth(&tg
->rt_bandwidth
,
207 ktime_to_ns(def_rt_bandwidth
.rt_period
), 0);
209 for_each_possible_cpu(i
) {
210 rt_rq
= kzalloc_node(sizeof(struct rt_rq
),
211 GFP_KERNEL
, cpu_to_node(i
));
215 rt_se
= kzalloc_node(sizeof(struct sched_rt_entity
),
216 GFP_KERNEL
, cpu_to_node(i
));
221 rt_rq
->rt_runtime
= tg
->rt_bandwidth
.rt_runtime
;
222 init_tg_rt_entry(tg
, rt_rq
, rt_se
, i
, parent
->rt_se
[i
]);
223 init_rt_entity_runnable_average(rt_se
);
234 #else /* CONFIG_RT_GROUP_SCHED */
236 #define rt_entity_is_task(rt_se) (1)
238 static inline struct task_struct
*rt_task_of(struct sched_rt_entity
*rt_se
)
240 return container_of(rt_se
, struct task_struct
, rt
);
243 static inline struct rq
*rq_of_rt_rq(struct rt_rq
*rt_rq
)
245 return container_of(rt_rq
, struct rq
, rt
);
248 static inline struct rq
*rq_of_rt_se(struct sched_rt_entity
*rt_se
)
250 struct task_struct
*p
= rt_task_of(rt_se
);
255 static inline struct rt_rq
*rt_rq_of_se(struct sched_rt_entity
*rt_se
)
257 struct rq
*rq
= rq_of_rt_se(rt_se
);
262 void free_rt_sched_group(struct task_group
*tg
) { }
264 int alloc_rt_sched_group(struct task_group
*tg
, struct task_group
*parent
)
268 #endif /* CONFIG_RT_GROUP_SCHED */
272 #include "sched-pelt.h"
273 #define entity_is_task(se) (!se->my_q)
275 extern u64
decay_load(u64 val
, u64 n
);
277 static u32
__accumulate_pelt_segments_rt(u64 periods
, u32 d1
, u32 d3
)
281 c1
= decay_load((u64
)d1
, periods
);
283 c2
= LOAD_AVG_MAX
- decay_load(LOAD_AVG_MAX
, periods
) - 1024;
288 #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT)
290 static __always_inline u32
291 accumulate_sum_rt(u64 delta
, int cpu
, struct sched_avg
*sa
,
292 unsigned long weight
, int running
)
294 unsigned long scale_freq
, scale_cpu
;
295 u32 contrib
= (u32
)delta
;
298 scale_freq
= arch_scale_freq_capacity(NULL
, cpu
);
299 scale_cpu
= arch_scale_cpu_capacity(NULL
, cpu
);
301 delta
+= sa
->period_contrib
;
302 periods
= delta
/ 1024;
305 sa
->load_sum
= decay_load(sa
->load_sum
, periods
);
306 sa
->util_sum
= decay_load((u64
)(sa
->util_sum
), periods
);
309 contrib
= __accumulate_pelt_segments_rt(periods
,
310 1024 - sa
->period_contrib
, delta
);
312 sa
->period_contrib
= delta
;
314 contrib
= cap_scale(contrib
, scale_freq
);
316 sa
->load_sum
+= weight
* contrib
;
319 sa
->util_sum
+= contrib
* scale_cpu
;
325 * We can represent the historical contribution to runnable average as the
326 * coefficients of a geometric series, exactly like fair task load.
327 * refer the ___update_load_avg @ fair sched class
329 static __always_inline
int
330 __update_load_avg(u64 now
, int cpu
, struct sched_avg
*sa
,
331 unsigned long weight
, int running
, struct rt_rq
*rt_rq
)
335 delta
= now
- sa
->last_update_time
;
337 if ((s64
)delta
< 0) {
338 sa
->last_update_time
= now
;
346 sa
->last_update_time
+= delta
<< 10;
351 if (!accumulate_sum_rt(delta
, cpu
, sa
, weight
, running
))
354 sa
->load_avg
= div_u64(sa
->load_sum
, LOAD_AVG_MAX
- 1024 + sa
->period_contrib
);
355 sa
->util_avg
= sa
->util_sum
/ (LOAD_AVG_MAX
- 1024 + sa
->period_contrib
);
360 static void pull_rt_task(struct rq
*this_rq
);
362 static inline bool need_pull_rt_task(struct rq
*rq
, struct task_struct
*prev
)
364 /* Try to pull RT tasks here if we lower this rq's prio */
365 return rq
->rt
.highest_prio
.curr
> prev
->prio
;
368 static inline int rt_overloaded(struct rq
*rq
)
370 return atomic_read(&rq
->rd
->rto_count
);
373 static inline void rt_set_overload(struct rq
*rq
)
378 cpumask_set_cpu(rq
->cpu
, rq
->rd
->rto_mask
);
380 * Make sure the mask is visible before we set
381 * the overload count. That is checked to determine
382 * if we should look at the mask. It would be a shame
383 * if we looked at the mask, but the mask was not
386 * Matched by the barrier in pull_rt_task().
389 atomic_inc(&rq
->rd
->rto_count
);
392 static inline void rt_clear_overload(struct rq
*rq
)
397 /* the order here really doesn't matter */
398 atomic_dec(&rq
->rd
->rto_count
);
399 cpumask_clear_cpu(rq
->cpu
, rq
->rd
->rto_mask
);
402 static void update_rt_migration(struct rt_rq
*rt_rq
)
404 if (rt_rq
->rt_nr_migratory
&& rt_rq
->rt_nr_total
> 1) {
405 if (!rt_rq
->overloaded
) {
406 rt_set_overload(rq_of_rt_rq(rt_rq
));
407 rt_rq
->overloaded
= 1;
409 } else if (rt_rq
->overloaded
) {
410 rt_clear_overload(rq_of_rt_rq(rt_rq
));
411 rt_rq
->overloaded
= 0;
415 static void inc_rt_migration(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
417 struct task_struct
*p
;
419 if (!rt_entity_is_task(rt_se
))
422 p
= rt_task_of(rt_se
);
423 rt_rq
= &rq_of_rt_rq(rt_rq
)->rt
;
425 rt_rq
->rt_nr_total
++;
426 if (p
->nr_cpus_allowed
> 1)
427 rt_rq
->rt_nr_migratory
++;
429 update_rt_migration(rt_rq
);
432 static void dec_rt_migration(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
434 struct task_struct
*p
;
436 if (!rt_entity_is_task(rt_se
))
439 p
= rt_task_of(rt_se
);
440 rt_rq
= &rq_of_rt_rq(rt_rq
)->rt
;
442 rt_rq
->rt_nr_total
--;
443 if (p
->nr_cpus_allowed
> 1)
444 rt_rq
->rt_nr_migratory
--;
446 update_rt_migration(rt_rq
);
449 static inline int has_pushable_tasks(struct rq
*rq
)
451 return !plist_head_empty(&rq
->rt
.pushable_tasks
);
454 static DEFINE_PER_CPU(struct callback_head
, rt_push_head
);
455 static DEFINE_PER_CPU(struct callback_head
, rt_pull_head
);
457 static void push_rt_tasks(struct rq
*);
458 static void pull_rt_task(struct rq
*);
460 static inline void queue_push_tasks(struct rq
*rq
)
462 if (!has_pushable_tasks(rq
))
465 queue_balance_callback(rq
, &per_cpu(rt_push_head
, rq
->cpu
), push_rt_tasks
);
468 static inline void queue_pull_task(struct rq
*rq
)
470 queue_balance_callback(rq
, &per_cpu(rt_pull_head
, rq
->cpu
), pull_rt_task
);
473 static void enqueue_pushable_task(struct rq
*rq
, struct task_struct
*p
)
475 plist_del(&p
->pushable_tasks
, &rq
->rt
.pushable_tasks
);
476 plist_node_init(&p
->pushable_tasks
, p
->prio
);
477 plist_add(&p
->pushable_tasks
, &rq
->rt
.pushable_tasks
);
479 /* Update the highest prio pushable task */
480 if (p
->prio
< rq
->rt
.highest_prio
.next
)
481 rq
->rt
.highest_prio
.next
= p
->prio
;
484 static void dequeue_pushable_task(struct rq
*rq
, struct task_struct
*p
)
486 plist_del(&p
->pushable_tasks
, &rq
->rt
.pushable_tasks
);
488 /* Update the new highest prio pushable task */
489 if (has_pushable_tasks(rq
)) {
490 p
= plist_first_entry(&rq
->rt
.pushable_tasks
,
491 struct task_struct
, pushable_tasks
);
492 rq
->rt
.highest_prio
.next
= p
->prio
;
494 rq
->rt
.highest_prio
.next
= MAX_RT_PRIO
;
499 static inline void enqueue_pushable_task(struct rq
*rq
, struct task_struct
*p
)
503 static inline void dequeue_pushable_task(struct rq
*rq
, struct task_struct
*p
)
508 void inc_rt_migration(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
513 void dec_rt_migration(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
517 static inline bool need_pull_rt_task(struct rq
*rq
, struct task_struct
*prev
)
522 static inline void pull_rt_task(struct rq
*this_rq
)
526 static inline void queue_push_tasks(struct rq
*rq
)
529 #endif /* CONFIG_SMP */
531 static void enqueue_top_rt_rq(struct rt_rq
*rt_rq
);
532 static void dequeue_top_rt_rq(struct rt_rq
*rt_rq
);
534 static inline int on_rt_rq(struct sched_rt_entity
*rt_se
)
539 #ifdef CONFIG_RT_GROUP_SCHED
541 static inline u64
sched_rt_runtime(struct rt_rq
*rt_rq
)
546 return rt_rq
->rt_runtime
;
549 static inline u64
sched_rt_period(struct rt_rq
*rt_rq
)
551 return ktime_to_ns(rt_rq
->tg
->rt_bandwidth
.rt_period
);
554 typedef struct task_group
*rt_rq_iter_t
;
556 static inline struct task_group
*next_task_group(struct task_group
*tg
)
559 tg
= list_entry_rcu(tg
->list
.next
,
560 typeof(struct task_group
), list
);
561 } while (&tg
->list
!= &task_groups
&& task_group_is_autogroup(tg
));
563 if (&tg
->list
== &task_groups
)
569 #define for_each_rt_rq(rt_rq, iter, rq) \
570 for (iter = container_of(&task_groups, typeof(*iter), list); \
571 (iter = next_task_group(iter)) && \
572 (rt_rq = iter->rt_rq[cpu_of(rq)]);)
574 #define for_each_sched_rt_entity(rt_se) \
575 for (; rt_se; rt_se = rt_se->parent)
577 static inline struct rt_rq
*group_rt_rq(struct sched_rt_entity
*rt_se
)
582 static void enqueue_rt_entity(struct sched_rt_entity
*rt_se
, unsigned int flags
);
583 static void dequeue_rt_entity(struct sched_rt_entity
*rt_se
, unsigned int flags
);
585 static void sched_rt_rq_enqueue(struct rt_rq
*rt_rq
)
587 struct task_struct
*curr
= rq_of_rt_rq(rt_rq
)->curr
;
588 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
589 struct sched_rt_entity
*rt_se
;
591 int cpu
= cpu_of(rq
);
593 rt_se
= rt_rq
->tg
->rt_se
[cpu
];
595 if (rt_rq
->rt_nr_running
) {
597 enqueue_top_rt_rq(rt_rq
);
598 else if (!on_rt_rq(rt_se
))
599 enqueue_rt_entity(rt_se
, 0);
601 if (rt_rq
->highest_prio
.curr
< curr
->prio
)
606 static void sched_rt_rq_dequeue(struct rt_rq
*rt_rq
)
608 struct sched_rt_entity
*rt_se
;
609 int cpu
= cpu_of(rq_of_rt_rq(rt_rq
));
611 rt_se
= rt_rq
->tg
->rt_se
[cpu
];
614 dequeue_top_rt_rq(rt_rq
);
615 else if (on_rt_rq(rt_se
))
616 dequeue_rt_entity(rt_se
, 0);
619 static inline int rt_rq_throttled(struct rt_rq
*rt_rq
)
621 return rt_rq
->rt_throttled
&& !rt_rq
->rt_nr_boosted
;
624 static int rt_se_boosted(struct sched_rt_entity
*rt_se
)
626 struct rt_rq
*rt_rq
= group_rt_rq(rt_se
);
627 struct task_struct
*p
;
630 return !!rt_rq
->rt_nr_boosted
;
632 p
= rt_task_of(rt_se
);
633 return p
->prio
!= p
->normal_prio
;
637 static inline const struct cpumask
*sched_rt_period_mask(void)
639 return this_rq()->rd
->span
;
642 static inline const struct cpumask
*sched_rt_period_mask(void)
644 return cpu_online_mask
;
649 struct rt_rq
*sched_rt_period_rt_rq(struct rt_bandwidth
*rt_b
, int cpu
)
651 return container_of(rt_b
, struct task_group
, rt_bandwidth
)->rt_rq
[cpu
];
654 static inline struct rt_bandwidth
*sched_rt_bandwidth(struct rt_rq
*rt_rq
)
656 return &rt_rq
->tg
->rt_bandwidth
;
659 #else /* !CONFIG_RT_GROUP_SCHED */
661 static inline u64
sched_rt_runtime(struct rt_rq
*rt_rq
)
663 return rt_rq
->rt_runtime
;
666 static inline u64
sched_rt_period(struct rt_rq
*rt_rq
)
668 return ktime_to_ns(def_rt_bandwidth
.rt_period
);
671 typedef struct rt_rq
*rt_rq_iter_t
;
673 #define for_each_rt_rq(rt_rq, iter, rq) \
674 for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
676 #define for_each_sched_rt_entity(rt_se) \
677 for (; rt_se; rt_se = NULL)
679 static inline struct rt_rq
*group_rt_rq(struct sched_rt_entity
*rt_se
)
684 static inline void sched_rt_rq_enqueue(struct rt_rq
*rt_rq
)
686 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
688 if (!rt_rq
->rt_nr_running
)
691 enqueue_top_rt_rq(rt_rq
);
695 static inline void sched_rt_rq_dequeue(struct rt_rq
*rt_rq
)
697 dequeue_top_rt_rq(rt_rq
);
700 static inline int rt_rq_throttled(struct rt_rq
*rt_rq
)
702 return rt_rq
->rt_throttled
;
705 static inline const struct cpumask
*sched_rt_period_mask(void)
707 return cpu_online_mask
;
711 struct rt_rq
*sched_rt_period_rt_rq(struct rt_bandwidth
*rt_b
, int cpu
)
713 return &cpu_rq(cpu
)->rt
;
716 static inline struct rt_bandwidth
*sched_rt_bandwidth(struct rt_rq
*rt_rq
)
718 return &def_rt_bandwidth
;
721 #endif /* CONFIG_RT_GROUP_SCHED */
723 bool sched_rt_bandwidth_account(struct rt_rq
*rt_rq
)
725 struct rt_bandwidth
*rt_b
= sched_rt_bandwidth(rt_rq
);
727 return (hrtimer_active(&rt_b
->rt_period_timer
) ||
728 rt_rq
->rt_time
< rt_b
->rt_runtime
);
733 * We ran out of runtime, see if we can borrow some from our neighbours.
735 static void do_balance_runtime(struct rt_rq
*rt_rq
)
737 struct rt_bandwidth
*rt_b
= sched_rt_bandwidth(rt_rq
);
738 struct root_domain
*rd
= rq_of_rt_rq(rt_rq
)->rd
;
742 weight
= cpumask_weight(rd
->span
);
744 raw_spin_lock(&rt_b
->rt_runtime_lock
);
745 rt_period
= ktime_to_ns(rt_b
->rt_period
);
746 for_each_cpu(i
, rd
->span
) {
747 struct rt_rq
*iter
= sched_rt_period_rt_rq(rt_b
, i
);
753 raw_spin_lock(&iter
->rt_runtime_lock
);
755 * Either all rqs have inf runtime and there's nothing to steal
756 * or __disable_runtime() below sets a specific rq to inf to
757 * indicate its been disabled and disalow stealing.
759 if (iter
->rt_runtime
== RUNTIME_INF
)
763 * From runqueues with spare time, take 1/n part of their
764 * spare time, but no more than our period.
766 diff
= iter
->rt_runtime
- iter
->rt_time
;
768 diff
= div_u64((u64
)diff
, weight
);
769 if (rt_rq
->rt_runtime
+ diff
> rt_period
)
770 diff
= rt_period
- rt_rq
->rt_runtime
;
771 iter
->rt_runtime
-= diff
;
772 rt_rq
->rt_runtime
+= diff
;
773 if (rt_rq
->rt_runtime
== rt_period
) {
774 raw_spin_unlock(&iter
->rt_runtime_lock
);
779 raw_spin_unlock(&iter
->rt_runtime_lock
);
781 raw_spin_unlock(&rt_b
->rt_runtime_lock
);
785 * Ensure this RQ takes back all the runtime it lend to its neighbours.
787 static void __disable_runtime(struct rq
*rq
)
789 struct root_domain
*rd
= rq
->rd
;
793 if (unlikely(!scheduler_running
))
796 for_each_rt_rq(rt_rq
, iter
, rq
) {
797 struct rt_bandwidth
*rt_b
= sched_rt_bandwidth(rt_rq
);
801 raw_spin_lock(&rt_b
->rt_runtime_lock
);
802 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
804 * Either we're all inf and nobody needs to borrow, or we're
805 * already disabled and thus have nothing to do, or we have
806 * exactly the right amount of runtime to take out.
808 if (rt_rq
->rt_runtime
== RUNTIME_INF
||
809 rt_rq
->rt_runtime
== rt_b
->rt_runtime
)
811 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
814 * Calculate the difference between what we started out with
815 * and what we current have, that's the amount of runtime
816 * we lend and now have to reclaim.
818 want
= rt_b
->rt_runtime
- rt_rq
->rt_runtime
;
821 * Greedy reclaim, take back as much as we can.
823 for_each_cpu(i
, rd
->span
) {
824 struct rt_rq
*iter
= sched_rt_period_rt_rq(rt_b
, i
);
828 * Can't reclaim from ourselves or disabled runqueues.
830 if (iter
== rt_rq
|| iter
->rt_runtime
== RUNTIME_INF
)
833 raw_spin_lock(&iter
->rt_runtime_lock
);
835 diff
= min_t(s64
, iter
->rt_runtime
, want
);
836 iter
->rt_runtime
-= diff
;
839 iter
->rt_runtime
-= want
;
842 raw_spin_unlock(&iter
->rt_runtime_lock
);
848 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
850 * We cannot be left wanting - that would mean some runtime
851 * leaked out of the system.
856 * Disable all the borrow logic by pretending we have inf
857 * runtime - in which case borrowing doesn't make sense.
859 rt_rq
->rt_runtime
= RUNTIME_INF
;
860 rt_rq
->rt_throttled
= 0;
861 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
862 raw_spin_unlock(&rt_b
->rt_runtime_lock
);
864 /* Make rt_rq available for pick_next_task() */
865 sched_rt_rq_enqueue(rt_rq
);
869 static void __enable_runtime(struct rq
*rq
)
874 if (unlikely(!scheduler_running
))
878 * Reset each runqueue's bandwidth settings
880 for_each_rt_rq(rt_rq
, iter
, rq
) {
881 struct rt_bandwidth
*rt_b
= sched_rt_bandwidth(rt_rq
);
883 raw_spin_lock(&rt_b
->rt_runtime_lock
);
884 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
885 rt_rq
->rt_runtime
= rt_b
->rt_runtime
;
887 rt_rq
->rt_throttled
= 0;
888 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
889 raw_spin_unlock(&rt_b
->rt_runtime_lock
);
893 static void balance_runtime(struct rt_rq
*rt_rq
)
895 if (!sched_feat(RT_RUNTIME_SHARE
))
898 if (rt_rq
->rt_time
> rt_rq
->rt_runtime
) {
899 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
900 do_balance_runtime(rt_rq
);
901 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
904 #else /* !CONFIG_SMP */
905 static inline void balance_runtime(struct rt_rq
*rt_rq
) {}
906 #endif /* CONFIG_SMP */
908 static int do_sched_rt_period_timer(struct rt_bandwidth
*rt_b
, int overrun
)
910 int i
, idle
= 1, throttled
= 0;
911 const struct cpumask
*span
;
913 span
= sched_rt_period_mask();
914 #ifdef CONFIG_RT_GROUP_SCHED
916 * FIXME: isolated CPUs should really leave the root task group,
917 * whether they are isolcpus or were isolated via cpusets, lest
918 * the timer run on a CPU which does not service all runqueues,
919 * potentially leaving other CPUs indefinitely throttled. If
920 * isolation is really required, the user will turn the throttle
921 * off to kill the perturbations it causes anyway. Meanwhile,
922 * this maintains functionality for boot and/or troubleshooting.
924 if (rt_b
== &root_task_group
.rt_bandwidth
)
925 span
= cpu_online_mask
;
927 for_each_cpu(i
, span
) {
929 struct rt_rq
*rt_rq
= sched_rt_period_rt_rq(rt_b
, i
);
930 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
934 * When span == cpu_online_mask, taking each rq->lock
935 * can be time-consuming. Try to avoid it when possible.
937 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
938 skip
= !rt_rq
->rt_time
&& !rt_rq
->rt_nr_running
;
939 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
943 raw_spin_lock(&rq
->lock
);
946 if (rt_rq
->rt_time
) {
949 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
950 if (rt_rq
->rt_throttled
)
951 balance_runtime(rt_rq
);
952 runtime
= rt_rq
->rt_runtime
;
953 rt_rq
->rt_time
-= min(rt_rq
->rt_time
, overrun
*runtime
);
954 if (rt_rq
->rt_throttled
&& rt_rq
->rt_time
< runtime
) {
955 rt_rq
->rt_throttled
= 0;
959 * When we're idle and a woken (rt) task is
960 * throttled check_preempt_curr() will set
961 * skip_update and the time between the wakeup
962 * and this unthrottle will get accounted as
965 if (rt_rq
->rt_nr_running
&& rq
->curr
== rq
->idle
)
966 rq_clock_skip_update(rq
, false);
968 if (rt_rq
->rt_time
|| rt_rq
->rt_nr_running
)
970 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
971 } else if (rt_rq
->rt_nr_running
) {
973 if (!rt_rq_throttled(rt_rq
))
976 if (rt_rq
->rt_throttled
)
980 sched_rt_rq_enqueue(rt_rq
);
981 raw_spin_unlock(&rq
->lock
);
984 if (!throttled
&& (!rt_bandwidth_enabled() || rt_b
->rt_runtime
== RUNTIME_INF
))
990 static inline int rt_se_prio(struct sched_rt_entity
*rt_se
)
992 #ifdef CONFIG_RT_GROUP_SCHED
993 struct rt_rq
*rt_rq
= group_rt_rq(rt_se
);
996 return rt_rq
->highest_prio
.curr
;
999 return rt_task_of(rt_se
)->prio
;
1002 static int sched_rt_runtime_exceeded(struct rt_rq
*rt_rq
)
1004 u64 runtime
= sched_rt_runtime(rt_rq
);
1006 if (rt_rq
->rt_throttled
)
1007 return rt_rq_throttled(rt_rq
);
1009 if (runtime
>= sched_rt_period(rt_rq
))
1012 balance_runtime(rt_rq
);
1013 runtime
= sched_rt_runtime(rt_rq
);
1014 if (runtime
== RUNTIME_INF
)
1017 if (rt_rq
->rt_time
> runtime
) {
1018 struct rt_bandwidth
*rt_b
= sched_rt_bandwidth(rt_rq
);
1021 * Don't actually throttle groups that have no runtime assigned
1022 * but accrue some time due to boosting.
1024 if (likely(rt_b
->rt_runtime
)) {
1025 rt_rq
->rt_throttled
= 1;
1026 printk_deferred_once("sched: RT throttling activated\n");
1029 * In case we did anyway, make it go away,
1030 * replenishment is a joke, since it will replenish us
1031 * with exactly 0 ns.
1036 if (rt_rq_throttled(rt_rq
)) {
1037 sched_rt_rq_dequeue(rt_rq
);
1046 * Update the current task's runtime statistics. Skip current tasks that
1047 * are not in our scheduling class.
1049 static void update_curr_rt(struct rq
*rq
)
1051 struct task_struct
*curr
= rq
->curr
;
1052 struct sched_rt_entity
*rt_se
= &curr
->rt
;
1055 if (curr
->sched_class
!= &rt_sched_class
)
1058 delta_exec
= rq_clock_task(rq
) - curr
->se
.exec_start
;
1059 if (unlikely((s64
)delta_exec
<= 0))
1062 /* Kick cpufreq (see the comment in kernel/sched/sched.h). */
1063 cpufreq_update_util(rq
, SCHED_CPUFREQ_RT
);
1065 schedstat_set(curr
->se
.statistics
.exec_max
,
1066 max(curr
->se
.statistics
.exec_max
, delta_exec
));
1068 curr
->se
.sum_exec_runtime
+= delta_exec
;
1069 account_group_exec_runtime(curr
, delta_exec
);
1071 curr
->se
.exec_start
= rq_clock_task(rq
);
1072 cpuacct_charge(curr
, delta_exec
);
1074 sched_rt_avg_update(rq
, delta_exec
);
1076 if (!rt_bandwidth_enabled())
1079 for_each_sched_rt_entity(rt_se
) {
1080 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
1082 if (sched_rt_runtime(rt_rq
) != RUNTIME_INF
) {
1083 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
1084 rt_rq
->rt_time
+= delta_exec
;
1085 if (sched_rt_runtime_exceeded(rt_rq
))
1087 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
1093 dequeue_top_rt_rq(struct rt_rq
*rt_rq
)
1095 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
1097 BUG_ON(&rq
->rt
!= rt_rq
);
1099 if (!rt_rq
->rt_queued
)
1102 BUG_ON(!rq
->nr_running
);
1104 sub_nr_running(rq
, rt_rq
->rt_nr_running
);
1105 rt_rq
->rt_queued
= 0;
1109 enqueue_top_rt_rq(struct rt_rq
*rt_rq
)
1111 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
1113 BUG_ON(&rq
->rt
!= rt_rq
);
1115 if (rt_rq
->rt_queued
)
1117 if (rt_rq_throttled(rt_rq
) || !rt_rq
->rt_nr_running
)
1120 add_nr_running(rq
, rt_rq
->rt_nr_running
);
1121 rt_rq
->rt_queued
= 1;
1124 #if defined CONFIG_SMP
1127 inc_rt_prio_smp(struct rt_rq
*rt_rq
, int prio
, int prev_prio
)
1129 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
1131 #ifdef CONFIG_RT_GROUP_SCHED
1133 * Change rq's cpupri only if rt_rq is the top queue.
1135 if (&rq
->rt
!= rt_rq
)
1138 if (rq
->online
&& prio
< prev_prio
)
1139 cpupri_set(&rq
->rd
->cpupri
, rq
->cpu
, prio
);
1143 dec_rt_prio_smp(struct rt_rq
*rt_rq
, int prio
, int prev_prio
)
1145 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
1147 #ifdef CONFIG_RT_GROUP_SCHED
1149 * Change rq's cpupri only if rt_rq is the top queue.
1151 if (&rq
->rt
!= rt_rq
)
1154 if (rq
->online
&& rt_rq
->highest_prio
.curr
!= prev_prio
)
1155 cpupri_set(&rq
->rd
->cpupri
, rq
->cpu
, rt_rq
->highest_prio
.curr
);
1158 #else /* CONFIG_SMP */
1161 void inc_rt_prio_smp(struct rt_rq
*rt_rq
, int prio
, int prev_prio
) {}
1163 void dec_rt_prio_smp(struct rt_rq
*rt_rq
, int prio
, int prev_prio
) {}
1165 #endif /* CONFIG_SMP */
1167 #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
1169 inc_rt_prio(struct rt_rq
*rt_rq
, int prio
)
1171 int prev_prio
= rt_rq
->highest_prio
.curr
;
1173 if (prio
< prev_prio
)
1174 rt_rq
->highest_prio
.curr
= prio
;
1176 inc_rt_prio_smp(rt_rq
, prio
, prev_prio
);
1180 dec_rt_prio(struct rt_rq
*rt_rq
, int prio
)
1182 int prev_prio
= rt_rq
->highest_prio
.curr
;
1184 if (rt_rq
->rt_nr_running
) {
1186 WARN_ON(prio
< prev_prio
);
1189 * This may have been our highest task, and therefore
1190 * we may have some recomputation to do
1192 if (prio
== prev_prio
) {
1193 struct rt_prio_array
*array
= &rt_rq
->active
;
1195 rt_rq
->highest_prio
.curr
=
1196 sched_find_first_bit(array
->bitmap
);
1200 rt_rq
->highest_prio
.curr
= MAX_RT_PRIO
;
1202 dec_rt_prio_smp(rt_rq
, prio
, prev_prio
);
1207 static inline void inc_rt_prio(struct rt_rq
*rt_rq
, int prio
) {}
1208 static inline void dec_rt_prio(struct rt_rq
*rt_rq
, int prio
) {}
1210 #endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */
1212 #ifdef CONFIG_RT_GROUP_SCHED
1215 inc_rt_group(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
1217 if (rt_se_boosted(rt_se
))
1218 rt_rq
->rt_nr_boosted
++;
1221 start_rt_bandwidth(&rt_rq
->tg
->rt_bandwidth
);
1225 dec_rt_group(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
1227 if (rt_se_boosted(rt_se
))
1228 rt_rq
->rt_nr_boosted
--;
1230 WARN_ON(!rt_rq
->rt_nr_running
&& rt_rq
->rt_nr_boosted
);
1233 #else /* CONFIG_RT_GROUP_SCHED */
1236 inc_rt_group(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
1238 start_rt_bandwidth(&def_rt_bandwidth
);
1242 void dec_rt_group(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
) {}
1244 #endif /* CONFIG_RT_GROUP_SCHED */
1247 unsigned int rt_se_nr_running(struct sched_rt_entity
*rt_se
)
1249 struct rt_rq
*group_rq
= group_rt_rq(rt_se
);
1252 return group_rq
->rt_nr_running
;
1258 unsigned int rt_se_rr_nr_running(struct sched_rt_entity
*rt_se
)
1260 struct rt_rq
*group_rq
= group_rt_rq(rt_se
);
1261 struct task_struct
*tsk
;
1264 return group_rq
->rr_nr_running
;
1266 tsk
= rt_task_of(rt_se
);
1268 return (tsk
->policy
== SCHED_RR
) ? 1 : 0;
1272 void inc_rt_tasks(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
1274 int prio
= rt_se_prio(rt_se
);
1276 WARN_ON(!rt_prio(prio
));
1277 rt_rq
->rt_nr_running
+= rt_se_nr_running(rt_se
);
1278 rt_rq
->rr_nr_running
+= rt_se_rr_nr_running(rt_se
);
1280 inc_rt_prio(rt_rq
, prio
);
1281 inc_rt_migration(rt_se
, rt_rq
);
1282 inc_rt_group(rt_se
, rt_rq
);
1286 void dec_rt_tasks(struct sched_rt_entity
*rt_se
, struct rt_rq
*rt_rq
)
1288 WARN_ON(!rt_prio(rt_se_prio(rt_se
)));
1289 WARN_ON(!rt_rq
->rt_nr_running
);
1290 rt_rq
->rt_nr_running
-= rt_se_nr_running(rt_se
);
1291 rt_rq
->rr_nr_running
-= rt_se_rr_nr_running(rt_se
);
1293 dec_rt_prio(rt_rq
, rt_se_prio(rt_se
));
1294 dec_rt_migration(rt_se
, rt_rq
);
1295 dec_rt_group(rt_se
, rt_rq
);
1300 attach_rt_entity_load_avg(struct rt_rq
*rt_rq
, struct sched_rt_entity
*rt_se
)
1302 rt_se
->avg
.last_update_time
= rt_rq
->avg
.last_update_time
;
1303 rt_rq
->avg
.util_avg
+= rt_se
->avg
.util_avg
;
1304 rt_rq
->avg
.util_sum
+= rt_se
->avg
.util_sum
;
1305 rt_rq
->avg
.load_avg
+= rt_se
->avg
.load_avg
;
1306 rt_rq
->avg
.load_sum
+= rt_se
->avg
.load_sum
;
1307 #ifdef CONFIG_RT_GROUP_SCHED
1308 rt_rq
->propagate_avg
= 1;
1310 rt_rq_util_change(rt_rq
);
1314 detach_rt_entity_load_avg(struct rt_rq
*rt_rq
, struct sched_rt_entity
*rt_se
)
1316 sub_positive(&rt_rq
->avg
.util_avg
, rt_se
->avg
.util_avg
);
1317 sub_positive(&rt_rq
->avg
.util_sum
, rt_se
->avg
.util_sum
);
1318 sub_positive(&rt_rq
->avg
.load_avg
, rt_se
->avg
.load_avg
);
1319 sub_positive(&rt_rq
->avg
.load_sum
, rt_se
->avg
.load_sum
);
1320 #ifdef CONFIG_RT_GROUP_SCHED
1321 rt_rq
->propagate_avg
= 1;
1323 rt_rq_util_change(rt_rq
);
1327 attach_rt_entity_load_avg(struct rt_rq
*rt_rq
, struct sched_rt_entity
*rt_se
) {}
1329 detach_rt_entity_load_avg(struct rt_rq
*rt_rq
, struct sched_rt_entity
*rt_se
) {}
1333 * Change rt_se->run_list location unless SAVE && !MOVE
1335 * assumes ENQUEUE/DEQUEUE flags match
1337 static inline bool move_entity(unsigned int flags
)
1339 if ((flags
& (DEQUEUE_SAVE
| DEQUEUE_MOVE
)) == DEQUEUE_SAVE
)
1345 static void __delist_rt_entity(struct sched_rt_entity
*rt_se
, struct rt_prio_array
*array
)
1347 list_del_init(&rt_se
->run_list
);
1349 if (list_empty(array
->queue
+ rt_se_prio(rt_se
)))
1350 __clear_bit(rt_se_prio(rt_se
), array
->bitmap
);
1355 static void __enqueue_rt_entity(struct sched_rt_entity
*rt_se
, unsigned int flags
)
1357 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
1358 struct rt_prio_array
*array
= &rt_rq
->active
;
1359 struct rt_rq
*group_rq
= group_rt_rq(rt_se
);
1360 struct list_head
*queue
= array
->queue
+ rt_se_prio(rt_se
);
1363 * Don't enqueue the group if its throttled, or when empty.
1364 * The latter is a consequence of the former when a child group
1365 * get throttled and the current group doesn't have any other
1368 if (group_rq
&& (rt_rq_throttled(group_rq
) || !group_rq
->rt_nr_running
)) {
1370 __delist_rt_entity(rt_se
, array
);
1374 if (move_entity(flags
)) {
1375 WARN_ON_ONCE(rt_se
->on_list
);
1376 if (flags
& ENQUEUE_HEAD
)
1377 list_add(&rt_se
->run_list
, queue
);
1379 list_add_tail(&rt_se
->run_list
, queue
);
1381 __set_bit(rt_se_prio(rt_se
), array
->bitmap
);
1386 update_rt_load_avg(rq_clock_task(rq_of_rt_rq(rt_rq
)), rt_se
);
1388 if (rt_entity_is_task(rt_se
) && !rt_se
->avg
.last_update_time
)
1389 attach_rt_entity_load_avg(rt_rq
, rt_se
);
1391 inc_rt_tasks(rt_se
, rt_rq
);
1394 static void __dequeue_rt_entity(struct sched_rt_entity
*rt_se
, unsigned int flags
)
1396 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
1397 struct rt_prio_array
*array
= &rt_rq
->active
;
1399 if (move_entity(flags
)) {
1400 WARN_ON_ONCE(!rt_se
->on_list
);
1401 __delist_rt_entity(rt_se
, array
);
1405 update_rt_load_avg(rq_clock_task(rq_of_rt_rq(rt_rq
)), rt_se
);
1407 dec_rt_tasks(rt_se
, rt_rq
);
1411 * Because the prio of an upper entry depends on the lower
1412 * entries, we must remove entries top - down.
1414 static void dequeue_rt_stack(struct sched_rt_entity
*rt_se
, unsigned int flags
)
1416 struct sched_rt_entity
*back
= NULL
;
1418 for_each_sched_rt_entity(rt_se
) {
1423 dequeue_top_rt_rq(rt_rq_of_se(back
));
1425 for (rt_se
= back
; rt_se
; rt_se
= rt_se
->back
) {
1426 if (on_rt_rq(rt_se
))
1427 __dequeue_rt_entity(rt_se
, flags
);
1431 static void enqueue_rt_entity(struct sched_rt_entity
*rt_se
, unsigned int flags
)
1433 struct rq
*rq
= rq_of_rt_se(rt_se
);
1435 dequeue_rt_stack(rt_se
, flags
);
1436 for_each_sched_rt_entity(rt_se
)
1437 __enqueue_rt_entity(rt_se
, flags
);
1438 enqueue_top_rt_rq(&rq
->rt
);
1441 static void dequeue_rt_entity(struct sched_rt_entity
*rt_se
, unsigned int flags
)
1443 struct rq
*rq
= rq_of_rt_se(rt_se
);
1445 dequeue_rt_stack(rt_se
, flags
);
1447 for_each_sched_rt_entity(rt_se
) {
1448 struct rt_rq
*rt_rq
= group_rt_rq(rt_se
);
1450 if (rt_rq
&& rt_rq
->rt_nr_running
)
1451 __enqueue_rt_entity(rt_se
, flags
);
1453 enqueue_top_rt_rq(&rq
->rt
);
1457 * Adding/removing a task to/from a priority array:
1460 enqueue_task_rt(struct rq
*rq
, struct task_struct
*p
, int flags
)
1462 struct sched_rt_entity
*rt_se
= &p
->rt
;
1464 if (flags
& ENQUEUE_WAKEUP
)
1467 enqueue_rt_entity(rt_se
, flags
);
1468 walt_inc_cumulative_runnable_avg(rq
, p
);
1470 if (!task_current(rq
, p
) && p
->nr_cpus_allowed
> 1)
1471 enqueue_pushable_task(rq
, p
);
1474 static void dequeue_task_rt(struct rq
*rq
, struct task_struct
*p
, int flags
)
1476 struct sched_rt_entity
*rt_se
= &p
->rt
;
1479 dequeue_rt_entity(rt_se
, flags
);
1480 walt_dec_cumulative_runnable_avg(rq
, p
);
1482 dequeue_pushable_task(rq
, p
);
1486 * Put task to the head or the end of the run list without the overhead of
1487 * dequeue followed by enqueue.
1490 requeue_rt_entity(struct rt_rq
*rt_rq
, struct sched_rt_entity
*rt_se
, int head
)
1492 if (on_rt_rq(rt_se
)) {
1493 struct rt_prio_array
*array
= &rt_rq
->active
;
1494 struct list_head
*queue
= array
->queue
+ rt_se_prio(rt_se
);
1497 list_move(&rt_se
->run_list
, queue
);
1499 list_move_tail(&rt_se
->run_list
, queue
);
1503 static void requeue_task_rt(struct rq
*rq
, struct task_struct
*p
, int head
)
1505 struct sched_rt_entity
*rt_se
= &p
->rt
;
1506 struct rt_rq
*rt_rq
;
1508 for_each_sched_rt_entity(rt_se
) {
1509 rt_rq
= rt_rq_of_se(rt_se
);
1510 requeue_rt_entity(rt_rq
, rt_se
, head
);
1514 static void yield_task_rt(struct rq
*rq
)
1516 requeue_task_rt(rq
, rq
->curr
, 0);
1522 * attach/detach/migrate_task_rt_rq() for load tracking
1525 #ifdef CONFIG_SCHED_USE_FLUID_RT
1526 static int find_lowest_rq(struct task_struct
*task
, int wake_flags
);
1528 static int find_lowest_rq(struct task_struct
*task
);
1531 select_task_rq_rt(struct task_struct
*p
, int cpu
, int sd_flag
, int flags
,
1532 int sibling_count_hint
)
1534 struct task_struct
*curr
;
1537 /* For anything but wake ups, just return the task_cpu */
1538 if (sd_flag
!= SD_BALANCE_WAKE
&& sd_flag
!= SD_BALANCE_FORK
)
1544 curr
= READ_ONCE(rq
->curr
); /* unlocked access */
1546 #ifdef CONFIG_SCHED_USE_FLUID_RT
1548 int target
= find_lowest_rq(p
, flags
);
1550 * Even though the destination CPU is running
1551 * a higher priority task, FluidRT can bother moving it
1552 * when its utilization is very small, and the other CPU is too busy
1553 * to accomodate the p in the point of priority and utilization.
1555 * BTW, if the curr has higher priority than p, FluidRT tries to find
1556 * the other CPUs first. In the worst case, curr can be victim, if it
1557 * has very small utilization.
1559 if (likely(target
!= -1)) {
1566 * If the current task on @p's runqueue is an RT task, then
1567 * try to see if we can wake this RT task up on another
1568 * runqueue. Otherwise simply start this RT task
1569 * on its current runqueue.
1571 * We want to avoid overloading runqueues. If the woken
1572 * task is a higher priority, then it will stay on this CPU
1573 * and the lower prio task should be moved to another CPU.
1574 * Even though this will probably make the lower prio task
1575 * lose its cache, we do not want to bounce a higher task
1576 * around just because it gave up its CPU, perhaps for a
1579 * For equal prio tasks, we just let the scheduler sort it out.
1581 * Otherwise, just let it ride on the affined RQ and the
1582 * post-schedule router will push the preempted task away
1584 * This test is optimistic, if we get it wrong the load-balancer
1585 * will have to sort it out.
1587 if (curr
&& unlikely(rt_task(curr
)) &&
1588 (curr
->nr_cpus_allowed
< 2 ||
1589 curr
->prio
<= p
->prio
)) {
1590 int target
= find_lowest_rq(p
);
1592 * Don't bother moving it if the destination CPU is
1593 * not running a lower priority task.
1596 p
->prio
< cpu_rq(target
)->rt
.highest_prio
.curr
)
1603 #ifdef CONFIG_SCHED_USE_FLUID_RT
1605 trace_sched_fluid_stat(p
, &p
->rt
.avg
, cpu
, "BIG_ASSIGED");
1610 #ifdef CONFIG_RT_GROUP_SCHED
1612 * Called within set_task_rq() right before setting a task's cpu. The
1613 * caller only guarantees p->pi_lock is held; no other assumptions,
1614 * including the state of rq->lock, should be made.
1616 void set_task_rq_rt(struct sched_rt_entity
*rt_se
,
1617 struct rt_rq
*prev
, struct rt_rq
*next
)
1619 u64 p_last_update_time
;
1620 u64 n_last_update_time
;
1622 if (!sched_feat(ATTACH_AGE_LOAD
))
1625 * We are supposed to update the task to "current" time, then its up to
1626 * date and ready to go to new CPU/rt_rq. But we have difficulty in
1627 * getting what current time is, so simply throw away the out-of-date
1628 * time. This will result in the wakee task is less decayed, but giving
1629 * the wakee more load sounds not bad.
1631 if (!(rt_se
->avg
.last_update_time
&& prev
))
1633 #ifndef CONFIG_64BIT
1635 u64 p_last_update_time_copy
;
1636 u64 n_last_update_time_copy
;
1639 p_last_update_time_copy
= prev
->load_last_update_time_copy
;
1640 n_last_update_time_copy
= next
->load_last_update_time_copy
;
1644 p_last_update_time
= prev
->avg
.last_update_time
;
1645 n_last_update_time
= next
->avg
.last_update_time
;
1647 } while (p_last_update_time
!= p_last_update_time_copy
||
1648 n_last_update_time
!= n_last_update_time_copy
);
1651 p_last_update_time
= prev
->avg
.last_update_time
;
1652 n_last_update_time
= next
->avg
.last_update_time
;
1654 __update_load_avg(p_last_update_time
, cpu_of(rq_of_rt_rq(prev
)),
1655 &rt_se
->avg
, 0, 0, NULL
);
1657 rt_se
->avg
.last_update_time
= n_last_update_time
;
1659 #endif /* CONFIG_RT_GROUP_SCHED */
1661 #ifndef CONFIG_64BIT
1662 static inline u64
rt_rq_last_update_time(struct rt_rq
*rt_rq
)
1664 u64 last_update_time_copy
;
1665 u64 last_update_time
;
1668 last_update_time_copy
= rt_rq
->load_last_update_time_copy
;
1670 last_update_time
= rt_rq
->avg
.last_update_time
;
1671 } while (last_update_time
!= last_update_time_copy
);
1673 return last_update_time
;
1676 static inline u64
rt_rq_last_update_time(struct rt_rq
*rt_rq
)
1678 return rt_rq
->avg
.last_update_time
;
1683 * Synchronize entity load avg of dequeued entity without locking
1686 void sync_rt_entity_load_avg(struct sched_rt_entity
*rt_se
)
1688 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
1689 u64 last_update_time
;
1691 last_update_time
= rt_rq_last_update_time(rt_rq
);
1692 __update_load_avg(last_update_time
, cpu_of(rq_of_rt_rq(rt_rq
)),
1693 &rt_se
->avg
, 0, 0, NULL
);
1697 * Task first catches up with rt_rq, and then subtract
1698 * itself from the rt_rq (task must be off the queue now).
1700 static void remove_rt_entity_load_avg(struct sched_rt_entity
*rt_se
)
1702 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
1705 * tasks cannot exit without having gone through wake_up_new_task() ->
1706 * post_init_entity_util_avg() which will have added things to the
1707 * rt_rq, so we can remove unconditionally.
1709 * Similarly for groups, they will have passed through
1710 * post_init_entity_util_avg() before unregister_sched_fair_group()
1714 sync_rt_entity_load_avg(rt_se
);
1715 atomic_long_add(rt_se
->avg
.load_avg
, &rt_rq
->removed_load_avg
);
1716 atomic_long_add(rt_se
->avg
.util_avg
, &rt_rq
->removed_util_avg
);
1719 static void attach_task_rt_rq(struct task_struct
*p
)
1721 struct sched_rt_entity
*rt_se
= &p
->rt
;
1722 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
1723 u64 now
= rq_clock_task(rq_of_rt_rq(rt_rq
));
1725 update_rt_load_avg(now
, rt_se
);
1726 attach_rt_entity_load_avg(rt_rq
, rt_se
);
1729 static void detach_task_rt_rq(struct task_struct
*p
)
1731 struct sched_rt_entity
*rt_se
= &p
->rt
;
1732 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
1733 u64 now
= rq_clock_task(rq_of_rt_rq(rt_rq
));
1735 update_rt_load_avg(now
, rt_se
);
1736 detach_rt_entity_load_avg(rt_rq
, rt_se
);
1739 static void migrate_task_rq_rt(struct task_struct
*p
)
1742 * We are supposed to update the task to "current" time, then its up to date
1743 * and ready to go to new CPU/cfs_rq. But we have difficulty in getting
1744 * what current time is, so simply throw away the out-of-date time. This
1745 * will result in the wakee task is less decayed, but giving the wakee more
1746 * load sounds not bad.
1748 remove_rt_entity_load_avg(&p
->rt
);
1750 /* Tell new CPU we are migrated */
1751 p
->rt
.avg
.last_update_time
= 0;
1753 /* We have migrated, no longer consider this task hot */
1754 p
->se
.exec_start
= 0;
1757 static void task_dead_rt(struct task_struct
*p
)
1759 remove_rt_entity_load_avg(&p
->rt
);
1762 #ifdef CONFIG_RT_GROUP_SCHED
1763 static void task_set_group_rt(struct task_struct
*p
)
1765 set_task_rq(p
, task_cpu(p
));
1768 static void task_move_group_rt(struct task_struct
*p
)
1770 detach_task_rt_rq(p
);
1771 set_task_rq(p
, task_cpu(p
));
1774 /* Tell se's cfs_rq has been changed -- migrated */
1775 p
->se
.avg
.last_update_time
= 0;
1777 attach_task_rt_rq(p
);
1780 static void task_change_group_rt(struct task_struct
*p
, int type
)
1783 case TASK_SET_GROUP
:
1784 task_set_group_rt(p
);
1787 case TASK_MOVE_GROUP
:
1788 task_move_group_rt(p
);
1794 static void check_preempt_equal_prio(struct rq
*rq
, struct task_struct
*p
)
1797 * Current can't be migrated, useless to reschedule,
1798 * let's hope p can move out.
1800 if (rq
->curr
->nr_cpus_allowed
== 1 ||
1801 !cpupri_find(&rq
->rd
->cpupri
, rq
->curr
, NULL
))
1805 * p is migratable, so let's not schedule it and
1806 * see if it is pushed or pulled somewhere else.
1808 if (p
->nr_cpus_allowed
!= 1
1809 && cpupri_find(&rq
->rd
->cpupri
, p
, NULL
))
1813 * There appears to be other cpus that can accept
1814 * current and none to run 'p', so lets reschedule
1815 * to try and push current away:
1817 requeue_task_rt(rq
, p
, 1);
1821 /* Give new sched_entity start runnable values to heavy its load in infant time */
1822 void init_rt_entity_runnable_average(struct sched_rt_entity
*rt_se
)
1824 struct sched_avg
*sa
= &rt_se
->avg
;
1826 sa
->last_update_time
= 0;
1828 sa
->period_contrib
= 1023;
1831 * Tasks are intialized with zero load.
1832 * Load is not actually used by RT, but can be inherited into fair task.
1837 * At this point, util_avg won't be used in select_task_rq_rt anyway
1841 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
1844 void init_rt_entity_runnable_average(struct sched_rt_entity
*rt_se
) { }
1845 #endif /* CONFIG_SMP */
1847 #ifdef CONFIG_SCHED_USE_FLUID_RT
1848 static inline void set_victim_flag(struct task_struct
*p
)
1853 static inline void clear_victim_flag(struct task_struct
*p
)
1858 static inline bool test_victim_flag(struct task_struct
*p
)
1866 static inline bool test_victim_flag(struct task_struct
*p
) { return false; }
1867 static inline void clear_victim_flag(struct task_struct
*p
) {}
1870 * Preempt the current task with a newly woken task if needed:
1872 static void check_preempt_curr_rt(struct rq
*rq
, struct task_struct
*p
, int flags
)
1874 if (p
->prio
< rq
->curr
->prio
) {
1877 } else if (test_victim_flag(p
)) {
1878 requeue_task_rt(rq
, p
, 1);
1887 * - the newly woken task is of equal priority to the current task
1888 * - the newly woken task is non-migratable while current is migratable
1889 * - current will be preempted on the next reschedule
1891 * we should check to see if current can readily move to a different
1892 * cpu. If so, we will reschedule to allow the push logic to try
1893 * to move current somewhere else, making room for our non-migratable
1896 if (p
->prio
== rq
->curr
->prio
&& !test_tsk_need_resched(rq
->curr
))
1897 check_preempt_equal_prio(rq
, p
);
1901 static struct sched_rt_entity
*pick_next_rt_entity(struct rq
*rq
,
1902 struct rt_rq
*rt_rq
)
1904 struct rt_prio_array
*array
= &rt_rq
->active
;
1905 struct sched_rt_entity
*next
= NULL
;
1906 struct list_head
*queue
;
1909 idx
= sched_find_first_bit(array
->bitmap
);
1910 BUG_ON(idx
>= MAX_RT_PRIO
);
1912 queue
= array
->queue
+ idx
;
1913 next
= list_entry(queue
->next
, struct sched_rt_entity
, run_list
);
1918 static struct task_struct
*_pick_next_task_rt(struct rq
*rq
)
1920 struct sched_rt_entity
*rt_se
;
1921 struct task_struct
*p
;
1922 struct rt_rq
*rt_rq
= &rq
->rt
;
1923 u64 now
= rq_clock_task(rq
);
1926 rt_se
= pick_next_rt_entity(rq
, rt_rq
);
1928 update_rt_load_avg(now
, rt_se
);
1929 rt_rq
->curr
= rt_se
;
1930 rt_rq
= group_rt_rq(rt_se
);
1933 p
= rt_task_of(rt_se
);
1934 p
->se
.exec_start
= now
;
1939 extern int update_rt_rq_load_avg(u64 now
, int cpu
, struct rt_rq
*rt_rq
, int running
);
1941 static struct task_struct
*
1942 pick_next_task_rt(struct rq
*rq
, struct task_struct
*prev
, struct rq_flags
*rf
)
1944 struct task_struct
*p
;
1945 struct rt_rq
*rt_rq
= &rq
->rt
;
1947 if (need_pull_rt_task(rq
, prev
)) {
1949 * This is OK, because current is on_cpu, which avoids it being
1950 * picked for load-balance and preemption/IRQs are still
1951 * disabled avoiding further scheduler activity on it and we're
1952 * being very careful to re-start the picking loop.
1954 rq_unpin_lock(rq
, rf
);
1956 rq_repin_lock(rq
, rf
);
1958 * pull_rt_task() can drop (and re-acquire) rq->lock; this
1959 * means a dl or stop task can slip in, in which case we need
1960 * to re-start task selection.
1962 if (unlikely((rq
->stop
&& task_on_rq_queued(rq
->stop
)) ||
1963 rq
->dl
.dl_nr_running
))
1968 * We may dequeue prev's rt_rq in put_prev_task().
1969 * So, we update time before rt_nr_running check.
1971 if (prev
->sched_class
== &rt_sched_class
)
1974 if (!rt_rq
->rt_queued
)
1977 put_prev_task(rq
, prev
);
1979 p
= _pick_next_task_rt(rq
);
1981 /* The running task is never eligible for pushing */
1982 dequeue_pushable_task(rq
, p
);
1984 queue_push_tasks(rq
);
1987 update_rt_rq_load_avg(rq_clock_task(rq
), cpu_of(rq
), rt_rq
,
1988 rq
->curr
->sched_class
== &rt_sched_class
);
1990 clear_victim_flag(p
);
1995 static void put_prev_task_rt(struct rq
*rq
, struct task_struct
*p
)
1997 struct sched_rt_entity
*rt_se
= &p
->rt
;
1998 u64 now
= rq_clock_task(rq
);
2003 * The previous task needs to be made eligible for pushing
2004 * if it is still active
2006 if (on_rt_rq(&p
->rt
) && p
->nr_cpus_allowed
> 1)
2007 enqueue_pushable_task(rq
, p
);
2009 for_each_sched_rt_entity(rt_se
) {
2010 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
2012 update_rt_load_avg(now
, rt_se
);
2020 void rt_rq_util_change(struct rt_rq
*rt_rq
)
2022 if (&this_rq()->rt
== rt_rq
)
2023 cpufreq_update_util(rt_rq
->rq
, SCHED_CPUFREQ_RT
);
2026 #ifdef CONFIG_RT_GROUP_SCHED
2027 /* Take into account change of utilization of a child task group */
2029 update_tg_rt_util(struct rt_rq
*cfs_rq
, struct sched_rt_entity
*rt_se
)
2031 struct rt_rq
*grt_rq
= rt_se
->my_q
;
2032 long delta
= grt_rq
->avg
.util_avg
- rt_se
->avg
.util_avg
;
2034 /* Nothing to update */
2038 /* Set new sched_rt_entity's utilization */
2039 rt_se
->avg
.util_avg
= grt_rq
->avg
.util_avg
;
2040 rt_se
->avg
.util_sum
= rt_se
->avg
.util_avg
* LOAD_AVG_MAX
;
2042 /* Update parent rt_rq utilization */
2043 add_positive(&cfs_rq
->avg
.util_avg
, delta
);
2044 cfs_rq
->avg
.util_sum
= cfs_rq
->avg
.util_avg
* LOAD_AVG_MAX
;
2048 /* Take into account change of load of a child task group */
2050 update_tg_rt_load(struct rt_rq
*rt_rq
, struct sched_rt_entity
*rt_se
)
2052 struct rt_rq
*grt_rq
= rt_se
->my_q
;
2053 long delta
= grt_rq
->avg
.load_avg
- rt_se
->avg
.load_avg
;
2056 * TODO: Need to consider the TG group update
2060 /* Nothing to update */
2064 /* Set new sched_rt_entity's load */
2065 rt_se
->avg
.load_avg
= grt_rq
->avg
.load_avg
;
2066 rt_se
->avg
.load_sum
= rt_se
->avg
.load_avg
* LOAD_AVG_MAX
;
2068 /* Update parent cfs_rq load */
2069 add_positive(&rt_rq
->avg
.load_avg
, delta
);
2070 rt_rq
->avg
.load_sum
= rt_rq
->avg
.load_avg
* LOAD_AVG_MAX
;
2073 * TODO: If the sched_entity is already enqueued, should we have to update the
2074 * runnable load avg.
2078 static inline int test_and_clear_tg_rt_propagate(struct sched_rt_entity
*rt_se
)
2080 struct rt_rq
*rt_rq
= rt_se
->my_q
;
2082 if (!rt_rq
->propagate_avg
)
2085 rt_rq
->propagate_avg
= 0;
2089 /* Update task and its cfs_rq load average */
2090 static inline int propagate_entity_rt_load_avg(struct sched_rt_entity
*rt_se
)
2092 struct rt_rq
*rt_rq
;
2094 if (rt_entity_is_task(rt_se
))
2097 if (!test_and_clear_tg_rt_propagate(rt_se
))
2100 rt_rq
= rt_rq_of_se(rt_se
);
2102 rt_rq
->propagate_avg
= 1;
2104 update_tg_rt_util(rt_rq
, rt_se
);
2105 update_tg_rt_load(rt_rq
, rt_se
);
2110 static inline int propagate_entity_rt_load_avg(struct sched_rt_entity
*rt_se
) { };
2113 void update_rt_load_avg(u64 now
, struct sched_rt_entity
*rt_se
)
2115 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
2116 struct rq
*rq
= rq_of_rt_rq(rt_rq
);
2117 int cpu
= cpu_of(rq
);
2119 * Track task load average for carrying it to new CPU after migrated.
2121 if (rt_se
->avg
.last_update_time
)
2122 __update_load_avg(now
, cpu
, &rt_se
->avg
, scale_load_down(NICE_0_LOAD
),
2123 rt_rq
->curr
== rt_se
, NULL
);
2125 update_rt_rq_load_avg(now
, cpu
, rt_rq
, true);
2126 propagate_entity_rt_load_avg(rt_se
);
2128 if (entity_is_task(rt_se
))
2129 trace_sched_rt_load_avg_task(rt_task_of(rt_se
), &rt_se
->avg
);
2132 /* Only try algorithms three times */
2133 #define RT_MAX_TRIES 3
2135 static int pick_rt_task(struct rq
*rq
, struct task_struct
*p
, int cpu
)
2137 if (!task_running(rq
, p
) &&
2138 cpumask_test_cpu(cpu
, &p
->cpus_allowed
))
2144 * Return the highest pushable rq's task, which is suitable to be executed
2145 * on the cpu, NULL otherwise
2147 static struct task_struct
*pick_highest_pushable_task(struct rq
*rq
, int cpu
)
2149 struct plist_head
*head
= &rq
->rt
.pushable_tasks
;
2150 struct task_struct
*p
;
2152 if (!has_pushable_tasks(rq
))
2155 plist_for_each_entry(p
, head
, pushable_tasks
) {
2156 if (pick_rt_task(rq
, p
, cpu
))
2163 static DEFINE_PER_CPU(cpumask_var_t
, local_cpu_mask
);
2165 #ifdef CONFIG_SCHED_USE_FLUID_RT
2166 static unsigned int sched_rt_boost_threshold
= 60;
2168 static inline struct cpumask
*sched_group_cpus_rt(struct sched_group
*sg
)
2170 return to_cpumask(sg
->cpumask
);
2173 static inline int weight_from_rtprio(int prio
)
2175 int idx
= (prio
>> 1);
2178 return sched_prio_to_weight
[prio
- MAX_RT_PRIO
];
2180 if ((idx
<< 1) == prio
)
2181 return rtprio_to_weight
[idx
];
2183 return ((rtprio_to_weight
[idx
] + rtprio_to_weight
[idx
+1]) >> 1);
2187 * to find the best CPU in which the data is kept in cache-hot
2189 * In most of time, RT task is invoked because,
2190 * Case - I : it is already scheduled some time ago, or
2191 * Case - II: it is requested by some task without timedelay
2193 * In case-I, it's hardly to find the best CPU in cache-hot if the time is relatively long.
2194 * But in case-II, waker CPU is likely to keep the cache-hot data useful to wakee RT task.
2196 static inline int affordable_cpu(int cpu
, unsigned long task_load
)
2199 * If the task.state is 'TASK_INTERRUPTIBLE',
2200 * she is likely to call 'schedule()' explicitely, for waking up RT task.
2201 * and have something in common with it.
2203 if (cpu_curr(cpu
)->state
!= TASK_INTERRUPTIBLE
)
2207 * Waker CPU must accommodate the target RT task.
2209 if (capacity_of(cpu
) <= task_load
)
2213 * Future work (More concerns if needed):
2214 * - Min opportunity cost between the eviction of tasks and dismiss of target RT
2215 * : If evicted tasks are expecting too many damage for its execution,
2216 * Target RT should not be this CPU.
2217 * load(RT) >= Capa(CPU)/3 && load(evicted tasks) >= Capa(CPU)/3
2218 * - Identifying the relation:
2219 * : Is it possible to identify the relation (such as mutex owner and waiter)
2226 extern unsigned long cpu_util_wake(int cpu
, struct task_struct
*p
);
2227 extern unsigned long task_util(struct task_struct
*p
);
2228 static inline int cpu_selected(int cpu
) { return (nr_cpu_ids
> cpu
&& cpu
>= 0); }
2230 * Must find the victim or recessive (not in lowest_mask)
2233 /* Future-safe accessor for struct task_struct's cpus_allowed. */
2234 #define rttsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
2236 static int find_victim_rt_rq(struct task_struct
*task
, const struct cpumask
*sg_cpus
, int *best_cpu
) {
2238 unsigned long victim_rtweight
, target_rtweight
, min_rtweight
;
2239 unsigned int victim_cpu_cap
, min_cpu_cap
= arch_scale_cpu_capacity(NULL
, task_cpu(task
));
2240 bool victim_rt
= true;
2245 target_rtweight
= task
->rt
.avg
.util_avg
* weight_from_rtprio(task
->prio
);
2246 min_rtweight
= target_rtweight
;
2248 for_each_cpu_and(i
, sg_cpus
, rttsk_cpus_allowed(task
)) {
2249 struct task_struct
*victim
= cpu_rq(i
)->curr
;
2251 if (victim
->nr_cpus_allowed
< 2)
2254 if (rt_task(victim
)) {
2255 victim_cpu_cap
= arch_scale_cpu_capacity(NULL
, i
);
2256 victim_rtweight
= victim
->rt
.avg
.util_avg
* weight_from_rtprio(victim
->prio
);
2258 if (min_cpu_cap
== victim_cpu_cap
) {
2259 if (victim_rtweight
< min_rtweight
) {
2260 min_rtweight
= victim_rtweight
;
2262 min_cpu_cap
= victim_cpu_cap
;
2266 * It's necessary to un-cap the cpu capacity when comparing
2267 * utilization of each CPU. This is why the Fluid RT tries to give
2268 * the green light on big CPU to the long-run RT task
2269 * in accordance with the priority.
2271 if (victim_rtweight
* min_cpu_cap
< min_rtweight
* victim_cpu_cap
) {
2272 min_rtweight
= victim_rtweight
;
2274 min_cpu_cap
= victim_cpu_cap
;
2278 /* If Non-RT CPU is exist, select it first. */
2285 if (*best_cpu
>= 0 && victim_rt
) {
2286 set_victim_flag(cpu_rq(*best_cpu
)->curr
);
2290 trace_sched_fluid_stat(task
, &task
->rt
.avg
, *best_cpu
, "VICTIM-FAIR");
2292 trace_sched_fluid_stat(task
, &task
->rt
.avg
, *best_cpu
, "VICTIM-RT");
2298 static int find_lowest_rq_fluid(struct task_struct
*task
, int wake_flags
)
2300 int cpu
, icpu
, best_cpu
= -1;
2301 int prefer_cpu
= smp_processor_id(); /* Cache-hot with itself or waker (default). */
2302 struct cpumask
*lowest_mask
= this_cpu_cpumask_var_ptr(local_cpu_mask
);
2304 u64 cpu_load
= ULLONG_MAX
, min_load
= ULLONG_MAX
, min_rt_load
= ULLONG_MAX
;
2305 u64 min_icl
= ULLONG_MAX
;
2306 int min_cpu
= -1, min_rt_cpu
= -1;
2308 /* Make sure the mask is initialized first */
2309 if (unlikely(!lowest_mask
)) {
2310 trace_sched_fluid_stat(task
, &task
->rt
.avg
, best_cpu
, "NA LOWESTMSK");
2314 if (task
->nr_cpus_allowed
== 1) {
2315 trace_sched_fluid_stat(task
, &task
->rt
.avg
, best_cpu
, "NA ALLOWED");
2316 goto out
; /* No other targets possible */
2319 /* update the per-cpu local_cpu_mask (lowest_mask) */
2320 cpupri_find(&task_rq(task
)->rd
->cpupri
, task
, lowest_mask
);
2324 * Fluid Sched Core selection procedure:
2326 * 1. Cache hot : this cpu (waker if wake_list is null)
2327 * 2. idle CPU selection (prev_cpu first)
2328 * 3. recessive task first (prev_cpu first)
2329 * 4. victim task first (prev_cpu first)
2333 * 1. Cache hot : packing the callee and caller,
2334 * when there is nothing to run except callee, or
2335 * wake_flags are set.
2337 /* FUTURE WORK: Hierarchical cache hot */
2338 if ((wake_flags
|| affordable_cpu(prefer_cpu
, task_util(task
))) &&
2339 cpumask_test_cpu(prefer_cpu
, cpu_online_mask
)) {
2340 task
->rt
.sync_flag
= 1;
2341 best_cpu
= prefer_cpu
;
2342 trace_sched_fluid_stat(task
, &task
->rt
.avg
, best_cpu
, "CACHE-HOT");
2347 * 2. idle CPU selection
2349 prefer_cpu
= task_cpu(task
);
2350 prefer_cpu
= (task
->rt
.avg
.util_avg
> sched_rt_boost_threshold
) ?
2351 cpumask_first(cpu_coregroup_mask(prefer_cpu
)) :
2352 cpumask_first(cpu_online_mask
);
2354 /* TODO: Need to refer the scheduling status of eHMP */
2355 for_each_online_cpu(cpu
){
2356 const struct cpumask
* traversingDom
= cpu_coregroup_mask(cpu
);
2357 if (cpu
!= cpumask_first(traversingDom
))
2360 if (cpumask_first(traversingDom
) < prefer_cpu
)
2363 for_each_cpu_and(icpu
, rttsk_cpus_allowed(task
), traversingDom
) {
2364 if (idle_cpu(icpu
)) {
2365 cpu_load
= cpu_util_wake(icpu
, task
) + task_util(task
);
2366 if ((min_icl
> cpu_load
) ||
2367 (min_icl
== cpu_load
&& task_cpu(task
) == icpu
)) {
2374 if (cpu_selected(best_cpu
)) {
2375 trace_sched_fluid_stat(task
, &task
->rt
.avg
, best_cpu
, "IDLE-FIRST");
2381 * 3. recessive task first
2383 prefer_cpu
= task_cpu(task
);
2385 for_each_online_cpu(cpu
) {
2386 if (cpu
!= cpumask_first(cpu_coregroup_mask(cpu
)))
2389 for_each_cpu_and(icpu
, rttsk_cpus_allowed(task
), cpu_coregroup_mask(cpu
)) {
2390 if (!cpumask_test_cpu(icpu
, lowest_mask
))
2393 cpu_load
= cpu_util_wake(icpu
, task
) + task_util(task
);
2395 if (rt_task(cpu_rq(icpu
)->curr
)) {
2396 if (cpu_load
< min_rt_load
||
2397 (cpu_load
== min_rt_load
&& icpu
== prefer_cpu
)) {
2398 min_rt_load
= cpu_load
;
2403 if (cpu_load
< min_load
||
2404 (cpu_load
== min_load
&& icpu
== prefer_cpu
)) {
2405 min_load
= cpu_load
;
2410 /* Fair recessive task : best min-load of non-rt cpu is exist? */
2411 if (cpu_selected(min_cpu
) &&
2412 ((capacity_of(min_cpu
) >= min_load
) || (min_cpu
== prefer_cpu
))) {
2414 trace_sched_fluid_stat(task
, &task
->rt
.avg
, best_cpu
, "FAIR-RECESS");
2418 /* RT recessive task : best min-load of rt cpu is exist? */
2419 if (cpu_selected(min_rt_cpu
) &&
2420 ((capacity_of(min_rt_cpu
) > min_rt_load
) || (min_rt_cpu
== prefer_cpu
))) {
2421 best_cpu
= min_rt_cpu
;
2422 trace_sched_fluid_stat(task
, &task
->rt
.avg
, best_cpu
, "RT-RECESS");
2428 * 4. victim task first
2430 for_each_online_cpu(cpu
) {
2431 if (cpu
!= cpumask_first(cpu_coregroup_mask(cpu
)))
2434 if (find_victim_rt_rq(task
, cpu_coregroup_mask(cpu
), &best_cpu
) != -1)
2438 if (!cpu_selected(best_cpu
))
2439 best_cpu
= prefer_cpu
;
2443 if (!cpumask_test_cpu(best_cpu
, cpu_online_mask
)) {
2444 trace_sched_fluid_stat(task
, &task
->rt
.avg
, cpu
, "NOTHING_VALID");
2450 #endif /* CONFIG_SCHED_USE_FLUID_RT */
2452 #ifdef CONFIG_SCHED_USE_FLUID_RT
2453 static int find_lowest_rq(struct task_struct
*task
, int wake_flags
)
2455 static int find_lowest_rq(struct task_struct
*task
)
2458 #ifdef CONFIG_SCHED_USE_FLUID_RT
2459 return find_lowest_rq_fluid(task
, wake_flags
);
2461 struct sched_domain
*sd
;
2462 struct cpumask
*lowest_mask
= this_cpu_cpumask_var_ptr(local_cpu_mask
);
2463 int this_cpu
= smp_processor_id();
2464 int cpu
= task_cpu(task
);
2466 /* Make sure the mask is initialized first */
2467 if (unlikely(!lowest_mask
))
2470 if (task
->nr_cpus_allowed
== 1)
2471 return -1; /* No other targets possible */
2473 if (!cpupri_find(&task_rq(task
)->rd
->cpupri
, task
, lowest_mask
))
2474 return -1; /* No targets found */
2477 * At this point we have built a mask of cpus representing the
2478 * lowest priority tasks in the system. Now we want to elect
2479 * the best one based on our affinity and topology.
2481 * We prioritize the last cpu that the task executed on since
2482 * it is most likely cache-hot in that location.
2484 if (cpumask_test_cpu(cpu
, lowest_mask
))
2488 * Otherwise, we consult the sched_domains span maps to figure
2489 * out which cpu is logically closest to our hot cache data.
2491 if (!cpumask_test_cpu(this_cpu
, lowest_mask
))
2492 this_cpu
= -1; /* Skip this_cpu opt if not among lowest */
2495 for_each_domain(cpu
, sd
) {
2496 if (sd
->flags
& SD_WAKE_AFFINE
) {
2500 * "this_cpu" is cheaper to preempt than a
2503 if (this_cpu
!= -1 &&
2504 cpumask_test_cpu(this_cpu
, sched_domain_span(sd
))) {
2509 best_cpu
= cpumask_first_and(lowest_mask
,
2510 sched_domain_span(sd
));
2511 if (best_cpu
< nr_cpu_ids
) {
2520 * And finally, if there were no matches within the domains
2521 * just give the caller *something* to work with from the compatible
2527 cpu
= cpumask_any(lowest_mask
);
2528 if (cpu
< nr_cpu_ids
)
2531 #endif /* CONFIG_SCHED_USE_FLUID_RT */
2534 /* Will lock the rq it finds */
2535 static struct rq
*find_lock_lowest_rq(struct task_struct
*task
, struct rq
*rq
)
2537 struct rq
*lowest_rq
= NULL
;
2541 for (tries
= 0; tries
< RT_MAX_TRIES
; tries
++) {
2542 #ifdef CONFIG_SCHED_USE_FLUID_RT
2543 cpu
= find_lowest_rq(task
, 0);
2545 cpu
= find_lowest_rq(task
);
2547 if ((cpu
== -1) || (cpu
== rq
->cpu
))
2550 lowest_rq
= cpu_rq(cpu
);
2551 if (lowest_rq
->rt
.highest_prio
.curr
<= task
->prio
)
2554 * Target rq has tasks of equal or higher priority,
2555 * retrying does not release any lock and is unlikely
2556 * to yield a different result.
2562 /* if the prio of this runqueue changed, try again */
2563 if (double_lock_balance(rq
, lowest_rq
)) {
2565 * We had to unlock the run queue. In
2566 * the mean time, task could have
2567 * migrated already or had its affinity changed.
2568 * Also make sure that it wasn't scheduled on its rq.
2570 if (unlikely(task_rq(task
) != rq
||
2571 !cpumask_test_cpu(lowest_rq
->cpu
, &task
->cpus_allowed
) ||
2572 task_running(rq
, task
) ||
2574 !task_on_rq_queued(task
))) {
2576 double_unlock_balance(rq
, lowest_rq
);
2582 /* If this rq is still suitable use it. */
2583 if (lowest_rq
->rt
.highest_prio
.curr
> task
->prio
)
2587 double_unlock_balance(rq
, lowest_rq
);
2594 static struct task_struct
*pick_next_pushable_task(struct rq
*rq
)
2596 struct task_struct
*p
;
2598 if (!has_pushable_tasks(rq
))
2601 p
= plist_first_entry(&rq
->rt
.pushable_tasks
,
2602 struct task_struct
, pushable_tasks
);
2604 BUG_ON(rq
->cpu
!= task_cpu(p
));
2605 BUG_ON(task_current(rq
, p
));
2606 BUG_ON(p
->nr_cpus_allowed
<= 1);
2608 BUG_ON(!task_on_rq_queued(p
));
2609 BUG_ON(!rt_task(p
));
2615 * If the current CPU has more than one RT task, see if the non
2616 * running task can migrate over to a CPU that is running a task
2617 * of lesser priority.
2619 static int push_rt_task(struct rq
*rq
)
2621 struct task_struct
*next_task
;
2622 struct rq
*lowest_rq
;
2625 if (!rq
->rt
.overloaded
)
2628 next_task
= pick_next_pushable_task(rq
);
2633 if (unlikely(next_task
== rq
->curr
)) {
2639 * It's possible that the next_task slipped in of
2640 * higher priority than current. If that's the case
2641 * just reschedule current.
2643 if (unlikely(next_task
->prio
< rq
->curr
->prio
)) {
2648 /* We might release rq lock */
2649 get_task_struct(next_task
);
2651 /* find_lock_lowest_rq locks the rq if found */
2652 lowest_rq
= find_lock_lowest_rq(next_task
, rq
);
2654 struct task_struct
*task
;
2656 * find_lock_lowest_rq releases rq->lock
2657 * so it is possible that next_task has migrated.
2659 * We need to make sure that the task is still on the same
2660 * run-queue and is also still the next task eligible for
2663 task
= pick_next_pushable_task(rq
);
2664 if (task
== next_task
) {
2666 * The task hasn't migrated, and is still the next
2667 * eligible task, but we failed to find a run-queue
2668 * to push it to. Do not retry in this case, since
2669 * other cpus will pull from us when ready.
2675 /* No more tasks, just exit */
2679 * Something has shifted, try again.
2681 put_task_struct(next_task
);
2686 deactivate_task(rq
, next_task
, 0);
2687 next_task
->on_rq
= TASK_ON_RQ_MIGRATING
;
2688 set_task_cpu(next_task
, lowest_rq
->cpu
);
2689 next_task
->on_rq
= TASK_ON_RQ_QUEUED
;
2690 activate_task(lowest_rq
, next_task
, 0);
2693 resched_curr(lowest_rq
);
2695 double_unlock_balance(rq
, lowest_rq
);
2698 put_task_struct(next_task
);
2703 static void push_rt_tasks(struct rq
*rq
)
2705 /* push_rt_task will return true if it moved an RT */
2706 while (push_rt_task(rq
))
2710 #ifdef HAVE_RT_PUSH_IPI
2713 * When a high priority task schedules out from a CPU and a lower priority
2714 * task is scheduled in, a check is made to see if there's any RT tasks
2715 * on other CPUs that are waiting to run because a higher priority RT task
2716 * is currently running on its CPU. In this case, the CPU with multiple RT
2717 * tasks queued on it (overloaded) needs to be notified that a CPU has opened
2718 * up that may be able to run one of its non-running queued RT tasks.
2720 * All CPUs with overloaded RT tasks need to be notified as there is currently
2721 * no way to know which of these CPUs have the highest priority task waiting
2722 * to run. Instead of trying to take a spinlock on each of these CPUs,
2723 * which has shown to cause large latency when done on machines with many
2724 * CPUs, sending an IPI to the CPUs to have them push off the overloaded
2725 * RT tasks waiting to run.
2727 * Just sending an IPI to each of the CPUs is also an issue, as on large
2728 * count CPU machines, this can cause an IPI storm on a CPU, especially
2729 * if its the only CPU with multiple RT tasks queued, and a large number
2730 * of CPUs scheduling a lower priority task at the same time.
2732 * Each root domain has its own irq work function that can iterate over
2733 * all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT
2734 * tassk must be checked if there's one or many CPUs that are lowering
2735 * their priority, there's a single irq work iterator that will try to
2736 * push off RT tasks that are waiting to run.
2738 * When a CPU schedules a lower priority task, it will kick off the
2739 * irq work iterator that will jump to each CPU with overloaded RT tasks.
2740 * As it only takes the first CPU that schedules a lower priority task
2741 * to start the process, the rto_start variable is incremented and if
2742 * the atomic result is one, then that CPU will try to take the rto_lock.
2743 * This prevents high contention on the lock as the process handles all
2744 * CPUs scheduling lower priority tasks.
2746 * All CPUs that are scheduling a lower priority task will increment the
2747 * rt_loop_next variable. This will make sure that the irq work iterator
2748 * checks all RT overloaded CPUs whenever a CPU schedules a new lower
2749 * priority task, even if the iterator is in the middle of a scan. Incrementing
2750 * the rt_loop_next will cause the iterator to perform another scan.
2753 static int rto_next_cpu(struct root_domain
*rd
)
2759 * When starting the IPI RT pushing, the rto_cpu is set to -1,
2760 * rt_next_cpu() will simply return the first CPU found in
2763 * If rto_next_cpu() is called with rto_cpu is a valid cpu, it
2764 * will return the next CPU found in the rto_mask.
2766 * If there are no more CPUs left in the rto_mask, then a check is made
2767 * against rto_loop and rto_loop_next. rto_loop is only updated with
2768 * the rto_lock held, but any CPU may increment the rto_loop_next
2769 * without any locking.
2773 /* When rto_cpu is -1 this acts like cpumask_first() */
2774 cpu
= cpumask_next(rd
->rto_cpu
, rd
->rto_mask
);
2778 if (cpu
< nr_cpu_ids
)
2784 * ACQUIRE ensures we see the @rto_mask changes
2785 * made prior to the @next value observed.
2787 * Matches WMB in rt_set_overload().
2789 next
= atomic_read_acquire(&rd
->rto_loop_next
);
2791 if (rd
->rto_loop
== next
)
2794 rd
->rto_loop
= next
;
2800 static inline bool rto_start_trylock(atomic_t
*v
)
2802 return !atomic_cmpxchg_acquire(v
, 0, 1);
2805 static inline void rto_start_unlock(atomic_t
*v
)
2807 atomic_set_release(v
, 0);
2810 static void tell_cpu_to_push(struct rq
*rq
)
2814 /* Keep the loop going if the IPI is currently active */
2815 atomic_inc(&rq
->rd
->rto_loop_next
);
2817 /* Only one CPU can initiate a loop at a time */
2818 if (!rto_start_trylock(&rq
->rd
->rto_loop_start
))
2821 raw_spin_lock(&rq
->rd
->rto_lock
);
2824 * The rto_cpu is updated under the lock, if it has a valid cpu
2825 * then the IPI is still running and will continue due to the
2826 * update to loop_next, and nothing needs to be done here.
2827 * Otherwise it is finishing up and an ipi needs to be sent.
2829 if (rq
->rd
->rto_cpu
< 0)
2830 cpu
= rto_next_cpu(rq
->rd
);
2832 raw_spin_unlock(&rq
->rd
->rto_lock
);
2834 rto_start_unlock(&rq
->rd
->rto_loop_start
);
2837 /* Make sure the rd does not get freed while pushing */
2838 sched_get_rd(rq
->rd
);
2839 irq_work_queue_on(&rq
->rd
->rto_push_work
, cpu
);
2843 /* Called from hardirq context */
2844 void rto_push_irq_work_func(struct irq_work
*work
)
2846 struct root_domain
*rd
=
2847 container_of(work
, struct root_domain
, rto_push_work
);
2854 * We do not need to grab the lock to check for has_pushable_tasks.
2855 * When it gets updated, a check is made if a push is possible.
2857 if (has_pushable_tasks(rq
)) {
2858 raw_spin_lock(&rq
->lock
);
2860 raw_spin_unlock(&rq
->lock
);
2863 raw_spin_lock(&rd
->rto_lock
);
2865 /* Pass the IPI to the next rt overloaded queue */
2866 cpu
= rto_next_cpu(rd
);
2868 raw_spin_unlock(&rd
->rto_lock
);
2875 /* Try the next RT overloaded CPU */
2876 irq_work_queue_on(&rd
->rto_push_work
, cpu
);
2878 #endif /* HAVE_RT_PUSH_IPI */
2880 static void pull_rt_task(struct rq
*this_rq
)
2882 int this_cpu
= this_rq
->cpu
, cpu
;
2883 bool resched
= false;
2884 struct task_struct
*p
;
2886 int rt_overload_count
= rt_overloaded(this_rq
);
2888 if (likely(!rt_overload_count
))
2892 * Match the barrier from rt_set_overloaded; this guarantees that if we
2893 * see overloaded we must also see the rto_mask bit.
2897 /* If we are the only overloaded CPU do nothing */
2898 if (rt_overload_count
== 1 &&
2899 cpumask_test_cpu(this_rq
->cpu
, this_rq
->rd
->rto_mask
))
2902 #ifdef HAVE_RT_PUSH_IPI
2903 if (sched_feat(RT_PUSH_IPI
)) {
2904 tell_cpu_to_push(this_rq
);
2909 for_each_cpu(cpu
, this_rq
->rd
->rto_mask
) {
2910 if (this_cpu
== cpu
)
2913 src_rq
= cpu_rq(cpu
);
2916 * Don't bother taking the src_rq->lock if the next highest
2917 * task is known to be lower-priority than our current task.
2918 * This may look racy, but if this value is about to go
2919 * logically higher, the src_rq will push this task away.
2920 * And if its going logically lower, we do not care
2922 if (src_rq
->rt
.highest_prio
.next
>=
2923 this_rq
->rt
.highest_prio
.curr
)
2927 * We can potentially drop this_rq's lock in
2928 * double_lock_balance, and another CPU could
2931 double_lock_balance(this_rq
, src_rq
);
2934 * We can pull only a task, which is pushable
2935 * on its rq, and no others.
2937 p
= pick_highest_pushable_task(src_rq
, this_cpu
);
2940 * Do we have an RT task that preempts
2941 * the to-be-scheduled task?
2943 if (p
&& (p
->prio
< this_rq
->rt
.highest_prio
.curr
)) {
2944 WARN_ON(p
== src_rq
->curr
);
2945 WARN_ON(!task_on_rq_queued(p
));
2948 * There's a chance that p is higher in priority
2949 * than what's currently running on its cpu.
2950 * This is just that p is wakeing up and hasn't
2951 * had a chance to schedule. We only pull
2952 * p if it is lower in priority than the
2953 * current task on the run queue
2955 if (p
->prio
< src_rq
->curr
->prio
)
2960 deactivate_task(src_rq
, p
, 0);
2961 p
->on_rq
= TASK_ON_RQ_MIGRATING
;
2962 set_task_cpu(p
, this_cpu
);
2963 p
->on_rq
= TASK_ON_RQ_QUEUED
;
2964 activate_task(this_rq
, p
, 0);
2966 * We continue with the search, just in
2967 * case there's an even higher prio task
2968 * in another runqueue. (low likelihood
2973 double_unlock_balance(this_rq
, src_rq
);
2977 resched_curr(this_rq
);
2981 * If we are not running and we are not going to reschedule soon, we should
2982 * try to push tasks away now
2984 static void task_woken_rt(struct rq
*rq
, struct task_struct
*p
)
2986 if (!task_running(rq
, p
) &&
2987 !test_tsk_need_resched(rq
->curr
) &&
2988 p
->nr_cpus_allowed
> 1 &&
2989 (dl_task(rq
->curr
) || rt_task(rq
->curr
)) &&
2990 (rq
->curr
->nr_cpus_allowed
< 2 ||
2991 rq
->curr
->prio
<= p
->prio
)) {
2992 #ifdef CONFIG_SCHED_USE_FLUID_RT
2993 if (p
->rt
.sync_flag
&& rq
->curr
->prio
< p
->prio
) {
2994 p
->rt
.sync_flag
= 0;
3001 #ifdef CONFIG_SCHED_USE_FLUID_RT
3002 p
->rt
.sync_flag
= 0;
3006 /* Assumes rq->lock is held */
3007 static void rq_online_rt(struct rq
*rq
)
3009 if (rq
->rt
.overloaded
)
3010 rt_set_overload(rq
);
3012 __enable_runtime(rq
);
3014 cpupri_set(&rq
->rd
->cpupri
, rq
->cpu
, rq
->rt
.highest_prio
.curr
);
3017 /* Assumes rq->lock is held */
3018 static void rq_offline_rt(struct rq
*rq
)
3020 if (rq
->rt
.overloaded
)
3021 rt_clear_overload(rq
);
3023 __disable_runtime(rq
);
3025 cpupri_set(&rq
->rd
->cpupri
, rq
->cpu
, CPUPRI_INVALID
);
3029 * When switch from the rt queue, we bring ourselves to a position
3030 * that we might want to pull RT tasks from other runqueues.
3032 static void switched_from_rt(struct rq
*rq
, struct task_struct
*p
)
3034 detach_task_rt_rq(p
);
3036 * If there are other RT tasks then we will reschedule
3037 * and the scheduling of the other RT tasks will handle
3038 * the balancing. But if we are the last RT task
3039 * we may need to handle the pulling of RT tasks
3042 if (!task_on_rq_queued(p
) || rq
->rt
.rt_nr_running
)
3045 queue_pull_task(rq
);
3048 void __init
init_sched_rt_class(void)
3052 for_each_possible_cpu(i
) {
3053 zalloc_cpumask_var_node(&per_cpu(local_cpu_mask
, i
),
3054 GFP_KERNEL
, cpu_to_node(i
));
3058 void update_rt_load_avg(u64 now
, struct sched_rt_entity
*rt_se
)
3061 #endif /* CONFIG_SMP */
3064 copy_sched_avg(struct sched_avg
*from
, struct sched_avg
*to
, unsigned int ratio
);
3067 * When switching a task to RT, we may overload the runqueue
3068 * with RT tasks. In this case we try to push them off to
3071 static void switched_to_rt(struct rq
*rq
, struct task_struct
*p
)
3073 /* Copy fair sched avg into rt sched avg */
3074 copy_sched_avg(&p
->se
.avg
, &p
->rt
.avg
, 100);
3076 * If we are already running, then there's nothing
3077 * that needs to be done. But if we are not running
3078 * we may need to preempt the current running task.
3079 * If that current running task is also an RT task
3080 * then see if we can move to another run queue.
3082 if (task_on_rq_queued(p
) && rq
->curr
!= p
) {
3084 if (p
->nr_cpus_allowed
> 1 && rq
->rt
.overloaded
)
3085 queue_push_tasks(rq
);
3086 #endif /* CONFIG_SMP */
3087 if (p
->prio
< rq
->curr
->prio
&& cpu_online(cpu_of(rq
)))
3093 * Priority of the task has changed. This may cause
3094 * us to initiate a push or pull.
3097 prio_changed_rt(struct rq
*rq
, struct task_struct
*p
, int oldprio
)
3099 if (!task_on_rq_queued(p
))
3102 if (rq
->curr
== p
) {
3105 * If our priority decreases while running, we
3106 * may need to pull tasks to this runqueue.
3108 if (oldprio
< p
->prio
)
3109 queue_pull_task(rq
);
3112 * If there's a higher priority task waiting to run
3115 if (p
->prio
> rq
->rt
.highest_prio
.curr
)
3118 /* For UP simply resched on drop of prio */
3119 if (oldprio
< p
->prio
)
3121 #endif /* CONFIG_SMP */
3124 * This task is not running, but if it is
3125 * greater than the current running task
3128 if (p
->prio
< rq
->curr
->prio
)
3133 #ifdef CONFIG_POSIX_TIMERS
3134 static void watchdog(struct rq
*rq
, struct task_struct
*p
)
3136 unsigned long soft
, hard
;
3138 /* max may change after cur was read, this will be fixed next tick */
3139 soft
= task_rlimit(p
, RLIMIT_RTTIME
);
3140 hard
= task_rlimit_max(p
, RLIMIT_RTTIME
);
3142 if (soft
!= RLIM_INFINITY
) {
3145 if (p
->rt
.watchdog_stamp
!= jiffies
) {
3147 p
->rt
.watchdog_stamp
= jiffies
;
3150 next
= DIV_ROUND_UP(min(soft
, hard
), USEC_PER_SEC
/HZ
);
3151 if (p
->rt
.timeout
> next
)
3152 p
->cputime_expires
.sched_exp
= p
->se
.sum_exec_runtime
;
3156 static inline void watchdog(struct rq
*rq
, struct task_struct
*p
) { }
3159 static void task_tick_rt(struct rq
*rq
, struct task_struct
*p
, int queued
)
3161 struct sched_rt_entity
*rt_se
= &p
->rt
;
3162 u64 now
= rq_clock_task(rq
);
3166 for_each_sched_rt_entity(rt_se
)
3167 update_rt_load_avg(now
, rt_se
);
3172 * RR tasks need a special form of timeslice management.
3173 * FIFO tasks have no timeslices.
3175 if (p
->policy
!= SCHED_RR
)
3178 if (--p
->rt
.time_slice
)
3181 p
->rt
.time_slice
= sched_rr_timeslice
;
3184 * Requeue to the end of queue if we (and all of our ancestors) are not
3185 * the only element on the queue
3187 for_each_sched_rt_entity(rt_se
) {
3188 if (rt_se
->run_list
.prev
!= rt_se
->run_list
.next
) {
3189 requeue_task_rt(rq
, p
, 0);
3196 static void set_curr_task_rt(struct rq
*rq
)
3198 struct task_struct
*p
= rq
->curr
;
3199 struct sched_rt_entity
*rt_se
= &p
->rt
;
3201 p
->se
.exec_start
= rq_clock_task(rq
);
3203 for_each_sched_rt_entity(rt_se
) {
3204 struct rt_rq
*rt_rq
= rt_rq_of_se(rt_se
);
3205 rt_rq
->curr
= rt_se
;
3208 /* The running task is never eligible for pushing */
3209 dequeue_pushable_task(rq
, p
);
3212 static unsigned int get_rr_interval_rt(struct rq
*rq
, struct task_struct
*task
)
3215 * Time slice is 0 for SCHED_FIFO tasks
3217 if (task
->policy
== SCHED_RR
)
3218 return sched_rr_timeslice
;
3223 const struct sched_class rt_sched_class
= {
3224 .next
= &fair_sched_class
,
3225 .enqueue_task
= enqueue_task_rt
,
3226 .dequeue_task
= dequeue_task_rt
,
3227 .yield_task
= yield_task_rt
,
3229 .check_preempt_curr
= check_preempt_curr_rt
,
3231 .pick_next_task
= pick_next_task_rt
,
3232 .put_prev_task
= put_prev_task_rt
,
3235 .select_task_rq
= select_task_rq_rt
,
3237 .migrate_task_rq
= migrate_task_rq_rt
,
3238 .task_dead
= task_dead_rt
,
3239 .set_cpus_allowed
= set_cpus_allowed_common
,
3240 .rq_online
= rq_online_rt
,
3241 .rq_offline
= rq_offline_rt
,
3242 .task_woken
= task_woken_rt
,
3243 .switched_from
= switched_from_rt
,
3246 .set_curr_task
= set_curr_task_rt
,
3247 .task_tick
= task_tick_rt
,
3249 .get_rr_interval
= get_rr_interval_rt
,
3251 .prio_changed
= prio_changed_rt
,
3252 .switched_to
= switched_to_rt
,
3254 .update_curr
= update_curr_rt
,
3255 #ifdef CONFIG_RT_GROUP_SCHED
3256 .task_change_group
= task_change_group_rt
,
3260 #ifdef CONFIG_RT_GROUP_SCHED
3262 * Ensure that the real time constraints are schedulable.
3264 static DEFINE_MUTEX(rt_constraints_mutex
);
3266 /* Must be called with tasklist_lock held */
3267 static inline int tg_has_rt_tasks(struct task_group
*tg
)
3269 struct task_struct
*g
, *p
;
3272 * Autogroups do not have RT tasks; see autogroup_create().
3274 if (task_group_is_autogroup(tg
))
3277 for_each_process_thread(g
, p
) {
3278 if (rt_task(p
) && task_group(p
) == tg
)
3285 struct rt_schedulable_data
{
3286 struct task_group
*tg
;
3291 static int tg_rt_schedulable(struct task_group
*tg
, void *data
)
3293 struct rt_schedulable_data
*d
= data
;
3294 struct task_group
*child
;
3295 unsigned long total
, sum
= 0;
3296 u64 period
, runtime
;
3298 period
= ktime_to_ns(tg
->rt_bandwidth
.rt_period
);
3299 runtime
= tg
->rt_bandwidth
.rt_runtime
;
3302 period
= d
->rt_period
;
3303 runtime
= d
->rt_runtime
;
3307 * Cannot have more runtime than the period.
3309 if (runtime
> period
&& runtime
!= RUNTIME_INF
)
3313 * Ensure we don't starve existing RT tasks.
3315 if (rt_bandwidth_enabled() && !runtime
&& tg_has_rt_tasks(tg
))
3318 total
= to_ratio(period
, runtime
);
3321 * Nobody can have more than the global setting allows.
3323 if (total
> to_ratio(global_rt_period(), global_rt_runtime()))
3327 * The sum of our children's runtime should not exceed our own.
3329 list_for_each_entry_rcu(child
, &tg
->children
, siblings
) {
3330 period
= ktime_to_ns(child
->rt_bandwidth
.rt_period
);
3331 runtime
= child
->rt_bandwidth
.rt_runtime
;
3333 if (child
== d
->tg
) {
3334 period
= d
->rt_period
;
3335 runtime
= d
->rt_runtime
;
3338 sum
+= to_ratio(period
, runtime
);
3347 static int __rt_schedulable(struct task_group
*tg
, u64 period
, u64 runtime
)
3351 struct rt_schedulable_data data
= {
3353 .rt_period
= period
,
3354 .rt_runtime
= runtime
,
3358 ret
= walk_tg_tree(tg_rt_schedulable
, tg_nop
, &data
);
3364 static int tg_set_rt_bandwidth(struct task_group
*tg
,
3365 u64 rt_period
, u64 rt_runtime
)
3370 * Disallowing the root group RT runtime is BAD, it would disallow the
3371 * kernel creating (and or operating) RT threads.
3373 if (tg
== &root_task_group
&& rt_runtime
== 0)
3376 /* No period doesn't make any sense. */
3380 mutex_lock(&rt_constraints_mutex
);
3381 read_lock(&tasklist_lock
);
3382 err
= __rt_schedulable(tg
, rt_period
, rt_runtime
);
3386 raw_spin_lock_irq(&tg
->rt_bandwidth
.rt_runtime_lock
);
3387 tg
->rt_bandwidth
.rt_period
= ns_to_ktime(rt_period
);
3388 tg
->rt_bandwidth
.rt_runtime
= rt_runtime
;
3390 for_each_possible_cpu(i
) {
3391 struct rt_rq
*rt_rq
= tg
->rt_rq
[i
];
3393 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
3394 rt_rq
->rt_runtime
= rt_runtime
;
3395 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
3397 raw_spin_unlock_irq(&tg
->rt_bandwidth
.rt_runtime_lock
);
3399 read_unlock(&tasklist_lock
);
3400 mutex_unlock(&rt_constraints_mutex
);
3405 int sched_group_set_rt_runtime(struct task_group
*tg
, long rt_runtime_us
)
3407 u64 rt_runtime
, rt_period
;
3409 rt_period
= ktime_to_ns(tg
->rt_bandwidth
.rt_period
);
3410 rt_runtime
= (u64
)rt_runtime_us
* NSEC_PER_USEC
;
3411 if (rt_runtime_us
< 0)
3412 rt_runtime
= RUNTIME_INF
;
3414 return tg_set_rt_bandwidth(tg
, rt_period
, rt_runtime
);
3417 long sched_group_rt_runtime(struct task_group
*tg
)
3421 if (tg
->rt_bandwidth
.rt_runtime
== RUNTIME_INF
)
3424 rt_runtime_us
= tg
->rt_bandwidth
.rt_runtime
;
3425 do_div(rt_runtime_us
, NSEC_PER_USEC
);
3426 return rt_runtime_us
;
3429 int sched_group_set_rt_period(struct task_group
*tg
, u64 rt_period_us
)
3431 u64 rt_runtime
, rt_period
;
3433 rt_period
= rt_period_us
* NSEC_PER_USEC
;
3434 rt_runtime
= tg
->rt_bandwidth
.rt_runtime
;
3436 return tg_set_rt_bandwidth(tg
, rt_period
, rt_runtime
);
3439 long sched_group_rt_period(struct task_group
*tg
)
3443 rt_period_us
= ktime_to_ns(tg
->rt_bandwidth
.rt_period
);
3444 do_div(rt_period_us
, NSEC_PER_USEC
);
3445 return rt_period_us
;
3448 static int sched_rt_global_constraints(void)
3452 mutex_lock(&rt_constraints_mutex
);
3453 read_lock(&tasklist_lock
);
3454 ret
= __rt_schedulable(NULL
, 0, 0);
3455 read_unlock(&tasklist_lock
);
3456 mutex_unlock(&rt_constraints_mutex
);
3461 int sched_rt_can_attach(struct task_group
*tg
, struct task_struct
*tsk
)
3463 /* Don't accept realtime tasks when there is no way for them to run */
3464 if (rt_task(tsk
) && tg
->rt_bandwidth
.rt_runtime
== 0)
3470 #else /* !CONFIG_RT_GROUP_SCHED */
3471 static int sched_rt_global_constraints(void)
3473 unsigned long flags
;
3476 raw_spin_lock_irqsave(&def_rt_bandwidth
.rt_runtime_lock
, flags
);
3477 for_each_possible_cpu(i
) {
3478 struct rt_rq
*rt_rq
= &cpu_rq(i
)->rt
;
3480 raw_spin_lock(&rt_rq
->rt_runtime_lock
);
3481 rt_rq
->rt_runtime
= global_rt_runtime();
3482 raw_spin_unlock(&rt_rq
->rt_runtime_lock
);
3484 raw_spin_unlock_irqrestore(&def_rt_bandwidth
.rt_runtime_lock
, flags
);
3488 #endif /* CONFIG_RT_GROUP_SCHED */
3490 static int sched_rt_global_validate(void)
3492 if (sysctl_sched_rt_period
<= 0)
3495 if ((sysctl_sched_rt_runtime
!= RUNTIME_INF
) &&
3496 (sysctl_sched_rt_runtime
> sysctl_sched_rt_period
))
3502 static void sched_rt_do_global(void)
3504 def_rt_bandwidth
.rt_runtime
= global_rt_runtime();
3505 def_rt_bandwidth
.rt_period
= ns_to_ktime(global_rt_period());
3508 int sched_rt_handler(struct ctl_table
*table
, int write
,
3509 void __user
*buffer
, size_t *lenp
,
3512 int old_period
, old_runtime
;
3513 static DEFINE_MUTEX(mutex
);
3517 old_period
= sysctl_sched_rt_period
;
3518 old_runtime
= sysctl_sched_rt_runtime
;
3520 ret
= proc_dointvec(table
, write
, buffer
, lenp
, ppos
);
3522 if (!ret
&& write
) {
3523 ret
= sched_rt_global_validate();
3527 ret
= sched_dl_global_validate();
3531 ret
= sched_rt_global_constraints();
3535 sched_rt_do_global();
3536 sched_dl_do_global();
3540 sysctl_sched_rt_period
= old_period
;
3541 sysctl_sched_rt_runtime
= old_runtime
;
3543 mutex_unlock(&mutex
);
3548 int sched_rr_handler(struct ctl_table
*table
, int write
,
3549 void __user
*buffer
, size_t *lenp
,
3553 static DEFINE_MUTEX(mutex
);
3556 ret
= proc_dointvec(table
, write
, buffer
, lenp
, ppos
);
3558 * Make sure that internally we keep jiffies.
3559 * Also, writing zero resets the timeslice to default:
3561 if (!ret
&& write
) {
3562 sched_rr_timeslice
=
3563 sysctl_sched_rr_timeslice
<= 0 ? RR_TIMESLICE
:
3564 msecs_to_jiffies(sysctl_sched_rr_timeslice
);
3566 mutex_unlock(&mutex
);
3570 #ifdef CONFIG_SCHED_DEBUG
3571 void print_rt_stats(struct seq_file
*m
, int cpu
)
3574 struct rt_rq
*rt_rq
;
3577 for_each_rt_rq(rt_rq
, iter
, cpu_rq(cpu
))
3578 print_rt_rq(m
, cpu
, rt_rq
);
3581 #endif /* CONFIG_SCHED_DEBUG */