sched/fair: add util_est on top of PELT

 - backport util-est from linux-power.git

The util_avg signal computed by PELT is too variable for some use-cases.
For example, a big task waking up after a long sleep period will have its
utilization almost completely decayed. This introduces some latency before
schedutil will be able to pick the best frequency to run a task.

The same issue can affect task placement. Indeed, since the task
utilization is already decayed at wakeup, when the task is enqueued in a
CPU, this can result in a CPU running a big task as being temporarily
represented as being almost empty. This leads to a race condition where
other tasks can be potentially allocated on a CPU which just started to run
a big task which slept for a relatively long period.

Moreover, the PELT utilization of a task can be updated every [ms], thus
making it a continuously changing value for certain longer running
tasks. This means that the instantaneous PELT utilization of a RUNNING
task is not really meaningful to properly support scheduler decisions.

For all these reasons, a more stable signal can do a better job of
representing the expected/estimated utilization of a task/cfs_rq.
Such a signal can be easily created on top of PELT by still using it as
an estimator which produces values to be aggregated on meaningful
events.

This patch adds a simple implementation of util_est, a new signal built on
top of PELT's util_avg where:

    util_est(task) = max(task::util_avg, f(task::util_avg@dequeue))

This allows to remember how big a task has been reported by PELT in its
previous activations via f(task::util_avg@dequeue), which is the new
_task_util_est(struct task_struct*) function added by this patch.

If a task should change its behavior and it runs longer in a new
activation, after a certain time its util_est will just track the
original PELT signal (i.e. task::util_avg).

The estimated utilization of cfs_rq is defined only for root ones.
That's because the only sensible consumer of this signal are the
scheduler and schedutil when looking for the overall CPU utilization
due to FAIR tasks.
For this reason, the estimated utilization of a root cfs_rq is simply
defined as:

    util_est(cfs_rq) = max(cfs_rq::util_avg, cfs_rq::util_est::enqueued)

where:

    cfs_rq::util_est::enqueued = sum(_task_util_est(task))
                                 for each RUNNABLE task on that root cfs_rq

It's worth to note that the estimated utilization is tracked only for
objects of interests, specifically:
 - Tasks: to better support tasks placement decisions
 - root cfs_rqs: to better support both tasks placement decisions as
                 well as frequencies selection

Change-Id: Ic5cad5aab372a8a247024e5304e4a55191fe16ea
Signed-off-by: Patrick Bellasi <patrick.bellasi@arm.com>
Reviewed-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Cc: Viresh Kumar <viresh.kumar@linaro.org>
Cc: Paul Turner <pjt@google.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: linux-kernel@vger.kernel.org
Cc: linux-pm@vger.kernel.org

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5ceb51cd3997b45c0488b9c5979273e843ad31b6..5bc3b25c406422fd0e8be74baaf5665154cc4de7 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -286,6 +286,34 @@ struct load_weight {
        u32                             inv_weight;
 };
 
+/**
+ * struct util_est - Estimation utilization of FAIR tasks
+ * @enqueued: instantaneous estimated utilization of a task/cpu
+ * @ewma:     the Exponential Weighted Moving Average (EWMA)
+ *            utilization of a task
+ *
+ * Support data structure to track an Exponential Weighted Moving Average
+ * (EWMA) of a FAIR task's utilization. New samples are added to the moving
+ * average each time a task completes an activation. Sample's weight is chosen
+ * so that the EWMA will be relatively insensitive to transient changes to the
+ * task's workload.
+ *
+ * The enqueued attribute has a slightly different meaning for tasks and cpus:
+ * - task:   the task's util_avg at last task dequeue time
+ * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
+ * Thus, the util_est.enqueued of a task represents the contribution on the
+ * estimated utilization of the CPU where that task is currently enqueued.
+ *
+ * Only for tasks we track a moving average of the past instantaneous
+ * estimated utilization. This allows to absorb sporadic drops in utilization
+ * of an otherwise almost periodic task.
+ */
+struct util_est {
+       unsigned int                    enqueued;
+       unsigned int                    ewma;
+#define UTIL_EST_WEIGHT_SHIFT          2
+};
+
 /*
  * The load_avg/util_avg accumulates an infinite geometric series
  * (see __update_load_avg() in kernel/sched/fair.c).
@@ -345,6 +373,7 @@ struct sched_avg {
        u32                             period_contrib;
        unsigned long                   load_avg;
        unsigned long                   util_avg;
+       struct util_est                 util_est;
 };
 
 #define ontime_of(p)                   (&p->se.ontime)

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 0a93f253673c2df477825d202c652bafe14b412e..4e75fda646c06ba6f2b39189b729888abc6b0072 100644 (file)
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -627,6 +627,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
                        cfs_rq->runnable_load_avg);
        SEQ_printf(m, "  .%-30s: %lu\n", "util_avg",
                        cfs_rq->avg.util_avg);
+       SEQ_printf(m, "  .%-30s: %u\n", "util_est_enqueued",
+                       cfs_rq->avg.util_est.enqueued);
        SEQ_printf(m, "  .%-30s: %ld\n", "removed_load_avg",
                        atomic_long_read(&cfs_rq->removed_load_avg));
        SEQ_printf(m, "  .%-30s: %ld\n", "removed_util_avg",
@@ -1073,6 +1075,8 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
        P(se.avg.load_avg);
        P(se.avg.util_avg);
        P(se.avg.last_update_time);
+       P(se.avg.util_est.ewma);
+       P(se.avg.util_est.enqueued);
 #endif
        P(policy);
        P(prio);

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 35a3aa4d8e9809883a7d649d8a1b0da03330884f..f05596eec7ccea9f017ee738dddacc7b6b314323 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3615,6 +3615,112 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
 
 static int idle_balance(struct rq *this_rq, struct rq_flags *rf);
 
+static inline unsigned long task_util(struct task_struct *p)
+{
+       return READ_ONCE(p->se.avg.util_avg);
+}
+
+static inline unsigned long _task_util_est(struct task_struct *p)
+{
+       struct util_est ue = READ_ONCE(p->se.avg.util_est);
+
+       return max(ue.ewma, ue.enqueued);
+}
+
+static inline unsigned long task_util_est(struct task_struct *p)
+{
+       return max(READ_ONCE(p->se.avg.util_avg), _task_util_est(p));
+}
+
+static inline void util_est_enqueue(struct cfs_rq *cfs_rq,
+                                   struct task_struct *p)
+{
+       unsigned int enqueued;
+
+       if (!sched_feat(UTIL_EST))
+               return;
+
+       /* Update root cfs_rq's estimated utilization */
+       enqueued  = cfs_rq->avg.util_est.enqueued;
+       enqueued += _task_util_est(p);
+       WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
+}
+
+/*
+ * Check if a (signed) value is within a specified (unsigned) margin,
+ * based on the observation that:
+ *     abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1)
+ *
+ * NOTE: this only works when value + maring < INT_MAX.
+ */
+static inline bool within_margin(int value, int margin)
+{
+       return ((unsigned int)(value + margin - 1) < (2 * margin - 1));
+}
+
+static void
+util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep)
+{
+       long last_ewma_diff;
+       struct util_est ue;
+
+       if (!sched_feat(UTIL_EST))
+               return;
+
+       /*
+        * Update root cfs_rq's estimated utilization
+        *
+        * If *p is the last task then the root cfs_rq's estimated utilization
+        * of a CPU is 0 by definition.
+        */
+       ue.enqueued = 0;
+       if (cfs_rq->nr_running) {
+               ue.enqueued  = cfs_rq->avg.util_est.enqueued;
+               ue.enqueued -= min_t(unsigned int, ue.enqueued,
+                                    _task_util_est(p));
+       }
+       WRITE_ONCE(cfs_rq->avg.util_est.enqueued, ue.enqueued);
+
+       /*
+        * Skip update of task's estimated utilization when the task has not
+        * yet completed an activation, e.g. being migrated.
+        */
+       if (!task_sleep)
+               return;
+
+       /*
+        * Skip update of task's estimated utilization when its EWMA is
+        * already ~1% close to its last activation value.
+        */
+       ue = p->se.avg.util_est;
+       ue.enqueued = task_util(p);
+       last_ewma_diff = ue.enqueued - ue.ewma;
+       if (within_margin(last_ewma_diff, (SCHED_CAPACITY_SCALE / 100)))
+               return;
+
+       /*
+        * Update Task's estimated utilization
+        *
+        * When *p completes an activation we can consolidate another sample
+        * of the task size. This is done by storing the current PELT value
+        * as ue.enqueued and by using this value to update the Exponential
+        * Weighted Moving Average (EWMA):
+        *
+        *  ewma(t) = w *  task_util(p) + (1-w) * ewma(t-1)
+        *          = w *  task_util(p) +         ewma(t-1)  - w * ewma(t-1)
+        *          = w * (task_util(p) -         ewma(t-1)) +     ewma(t-1)
+        *          = w * (      last_ewma_diff            ) +     ewma(t-1)
+        *          = w * (last_ewma_diff  +  ewma(t-1) / w)
+        *
+        * Where 'w' is the weight of new samples, which is configured to be
+        * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
+        */
+       ue.ewma <<= UTIL_EST_WEIGHT_SHIFT;
+       ue.ewma  += last_ewma_diff;
+       ue.ewma >>= UTIL_EST_WEIGHT_SHIFT;
+       WRITE_ONCE(p->se.avg.util_est, ue);
+}
+
 #else /* CONFIG_SMP */
 
 static inline int
@@ -3652,6 +3758,13 @@ static inline int idle_balance(struct rq *rq, struct rq_flags *rf)
        return 0;
 }
 
+static inline void
+util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
+
+static inline void
+util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p,
+                bool task_sleep) {}
+
 #endif /* CONFIG_SMP */
 
 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -5074,6 +5187,8 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
                walt_inc_cumulative_runnable_avg(rq, p);
        }
 
+
+       util_est_enqueue(&rq->cfs, p);
        hrtick_update(rq);
 }
 
@@ -5146,6 +5261,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
                walt_dec_cumulative_runnable_avg(rq, p);
        }
 
+       util_est_dequeue(&rq->cfs, p, task_sleep);
        hrtick_update(rq);
 }
 

diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 738f5914a6b60a4d0aaec1605d993d3687ad6a79..ea53adbf81bdeb4dad96115ca7c041f2f76130cb 100644 (file)
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -92,6 +92,11 @@ SCHED_FEAT(WA_BIAS, true)
  */
 #ifdef CONFIG_DEFAULT_USE_ENERGY_AWARE
 SCHED_FEAT(ENERGY_AWARE, true)
+
+/*
+ * UtilEstimation. Use estimated CPU utilization.
+ */
+SCHED_FEAT(UTIL_EST, true)
 #else
 SCHED_FEAT(ENERGY_AWARE, false)
 #endif