UPSTREAM: delayacct: track delays from thrashing cache pages

Delay accounting already measures the time a task spends in direct reclaim
and waiting for swapin, but in low memory situations tasks spend can spend
a significant amount of their time waiting on thrashing page cache.  This
isn't tracked right now.

To know the full impact of memory contention on an individual task,
measure the delay when waiting for a recently evicted active cache page to
read back into memory.

Also update tools/accounting/getdelays.c:

     [hannes@computer accounting]$ sudo ./getdelays -d -p 1
     print delayacct stats ON
     PID     1

     CPU             count     real total  virtual total    delay total  delay average
                     50318      745000000      847346785      400533713          0.008ms
     IO              count    delay total  delay average
                       435      122601218              0ms
     SWAP            count    delay total  delay average
                         0              0              0ms
     RECLAIM         count    delay total  delay average
                         0              0              0ms
     THRASHING       count    delay total  delay average
                        19       12621439              0ms

Link: http://lkml.kernel.org/r/20180828172258.3185-4-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: Daniel Drake <drake@endlessm.com>
Tested-by: Suren Baghdasaryan <surenb@google.com>
Cc: Christopher Lameter <cl@linux.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Johannes Weiner <jweiner@fb.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Enderborg <peter.enderborg@sony.com>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vinayak Menon <vinmenon@codeaurora.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
(cherry picked from commit b1d29ba82cf2bc784f4c963ddd6a2cf29e229b33)

Bug: 127712811
Test: lmkd in PSI mode
Change-Id: I259f693987cf04e6a52ee7e8accf55a17e0de005
Signed-off-by: Suren Baghdasaryan <surenb@google.com>

diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h
index 31c865d1842e88671d7f29534587311c479fb3e7..577d1b25fccdb08bcd152d1b2df3b26bbdc02864 100644 (file)
--- a/include/linux/delayacct.h
+++ b/include/linux/delayacct.h
@@ -57,7 +57,12 @@ struct task_delay_info {
 
        u64 freepages_start;
        u64 freepages_delay;    /* wait for memory reclaim */
+
+       u64 thrashing_start;
+       u64 thrashing_delay;    /* wait for thrashing page */
+
        u32 freepages_count;    /* total count of memory reclaim */
+       u32 thrashing_count;    /* total count of thrash waits */
 };
 #endif
 
@@ -76,6 +81,8 @@ extern int __delayacct_add_tsk(struct taskstats *, struct task_struct *);
 extern __u64 __delayacct_blkio_ticks(struct task_struct *);
 extern void __delayacct_freepages_start(void);
 extern void __delayacct_freepages_end(void);
+extern void __delayacct_thrashing_start(void);
+extern void __delayacct_thrashing_end(void);
 
 static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
 {
@@ -156,6 +163,18 @@ static inline void delayacct_freepages_end(void)
                __delayacct_freepages_end();
 }
 
+static inline void delayacct_thrashing_start(void)
+{
+       if (current->delays)
+               __delayacct_thrashing_start();
+}
+
+static inline void delayacct_thrashing_end(void)
+{
+       if (current->delays)
+               __delayacct_thrashing_end();
+}
+
 #else
 static inline void delayacct_set_flag(int flag)
 {}
@@ -182,6 +201,10 @@ static inline void delayacct_freepages_start(void)
 {}
 static inline void delayacct_freepages_end(void)
 {}
+static inline void delayacct_thrashing_start(void)
+{}
+static inline void delayacct_thrashing_end(void)
+{}
 
 #endif /* CONFIG_TASK_DELAY_ACCT */
 

diff --git a/include/uapi/linux/taskstats.h b/include/uapi/linux/taskstats.h
index b7aa7bb2349f7b72e3a746b3f96aef60266a21e6..5e8ca16a9079717886261f63c3285edddecaac5a 100644 (file)
--- a/include/uapi/linux/taskstats.h
+++ b/include/uapi/linux/taskstats.h
@@ -34,7 +34,7 @@
  */
 
 
-#define TASKSTATS_VERSION      8
+#define TASKSTATS_VERSION      9
 #define TS_COMM_LEN            32      /* should be >= TASK_COMM_LEN
                                         * in linux/sched.h */
 
@@ -164,6 +164,10 @@ struct taskstats {
        /* Delay waiting for memory reclaim */
        __u64   freepages_count;
        __u64   freepages_delay_total;
+
+       /* Delay waiting for thrashing page */
+       __u64   thrashing_count;
+       __u64   thrashing_delay_total;
 };
 
 

diff --git a/kernel/delayacct.c b/kernel/delayacct.c
index ca8ac2824f0b666d880154602f313db7e4112b42..2a12b988c7178a2ed2ed211745a8c3c4840f7955 100644 (file)
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -135,9 +135,12 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
        d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp;
        tmp = d->freepages_delay_total + tsk->delays->freepages_delay;
        d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp;
+       tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay;
+       d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp;
        d->blkio_count += tsk->delays->blkio_count;
        d->swapin_count += tsk->delays->swapin_count;
        d->freepages_count += tsk->delays->freepages_count;
+       d->thrashing_count += tsk->delays->thrashing_count;
        raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
 
        return 0;
@@ -169,3 +172,15 @@ void __delayacct_freepages_end(void)
                &current->delays->freepages_count);
 }
 
+void __delayacct_thrashing_start(void)
+{
+       current->delays->thrashing_start = ktime_get_ns();
+}
+
+void __delayacct_thrashing_end(void)
+{
+       delayacct_end(&current->delays->lock,
+                     &current->delays->thrashing_start,
+                     &current->delays->thrashing_delay,
+                     &current->delays->thrashing_count);
+}

diff --git a/mm/filemap.c b/mm/filemap.c
index c931dea6bdb5fa3d6255fe544fbdb40f6db93051..0aef9d24a69b23e6d033baf4327d86cba5c753e0 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -36,6 +36,7 @@
 #include <linux/memcontrol.h>
 #include <linux/cleancache.h>
 #include <linux/rmap.h>
+#include <linux/delayacct.h>
 #include "internal.h"
 
 #define CREATE_TRACE_POINTS
@@ -975,8 +976,15 @@ static inline int wait_on_page_bit_common(wait_queue_head_t *q,
 {
        struct wait_page_queue wait_page;
        wait_queue_entry_t *wait = &wait_page.wait;
+       bool thrashing = false;
        int ret = 0;
 
+       if (bit_nr == PG_locked && !PageSwapBacked(page) &&
+           !PageUptodate(page) && PageWorkingset(page)) {
+               delayacct_thrashing_start();
+               thrashing = true;
+       }
+
        init_wait(wait);
        wait->flags = lock ? WQ_FLAG_EXCLUSIVE : 0;
        wait->func = wake_page_function;
@@ -1015,6 +1023,9 @@ static inline int wait_on_page_bit_common(wait_queue_head_t *q,
 
        finish_wait(q, wait);
 
+       if (thrashing)
+               delayacct_thrashing_end();
+
        /*
         * A signal could leave PageWaiters set. Clearing it here if
         * !waitqueue_active would be possible (by open-coding finish_wait),

diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c
index 9f420d98b5fb3b2ca364ed6aff37908d42519333..8cb504d3038447fe4cbb2f033d6002e8b714b252 100644 (file)
--- a/tools/accounting/getdelays.c
+++ b/tools/accounting/getdelays.c
@@ -203,6 +203,8 @@ static void print_delayacct(struct taskstats *t)
               "SWAP  %15s%15s%15s\n"
               "      %15llu%15llu%15llums\n"
               "RECLAIM  %12s%15s%15s\n"
+              "      %15llu%15llu%15llums\n"
+              "THRASHING%12s%15s%15s\n"
               "      %15llu%15llu%15llums\n",
               "count", "real total", "virtual total",
               "delay total", "delay average",
@@ -222,7 +224,11 @@ static void print_delayacct(struct taskstats *t)
               "count", "delay total", "delay average",
               (unsigned long long)t->freepages_count,
               (unsigned long long)t->freepages_delay_total,
-              average_ms(t->freepages_delay_total, t->freepages_count));
+              average_ms(t->freepages_delay_total, t->freepages_count),
+              "count", "delay total", "delay average",
+              (unsigned long long)t->thrashing_count,
+              (unsigned long long)t->thrashing_delay_total,
+              average_ms(t->thrashing_delay_total, t->thrashing_count));
 }
 
 static void task_context_switch_counts(struct taskstats *t)