FROMLIST: [PATCH v5 01/12] arm: vdso: rename vdso_datapage variables

(cherry picked from url https://patchwork.kernel.org/patch/10044505/)

Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.

Rename seq_count to tb_seq_count. Rename tk_is_cntvct to use_syscall.
Rename cs_mult to cs_mono_mult. All to align with the variables in the
arm64 vdso datapage. Rework vdso_read_begin() and vdso_read_retry()
functions to reflect modern access patterns for tb_seq_count field.

Update copyright message to reflect the start of the contributions in
this series.

Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Bug: 63737556
Bug: 20045882
Change-Id: I13f16e71b1ecba3d72b999caafef72e3c7f48dfe

diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h
index 9be259442fca011464a1dc001a2f56b728d815ff..fa3e1856244dd439bf9fd558ebc1e2541bfa0799 100644 (file)
--- a/arch/arm/include/asm/vdso_datapage.h
+++ b/arch/arm/include/asm/vdso_datapage.h
@@ -29,8 +29,8 @@
  * 32 bytes.
  */
 struct vdso_data {
-       u32 seq_count;          /* sequence count - odd during updates */
-       u16 tk_is_cntvct;       /* fall back to syscall if false */
+       u32 tb_seq_count;       /* sequence count - odd during updates */
+       u16 use_syscall;        /* fall back to syscall if true */
        u16 cs_shift;           /* clocksource shift */
        u32 xtime_coarse_sec;   /* coarse time */
        u32 xtime_coarse_nsec;
@@ -38,7 +38,7 @@ struct vdso_data {
        u32 wtm_clock_sec;      /* wall to monotonic offset */
        u32 wtm_clock_nsec;
        u32 xtime_clock_sec;    /* CLOCK_REALTIME - seconds */
-       u32 cs_mult;            /* clocksource multiplier */
+       u32 cs_mono_mult;       /* clocksource multiplier */
 
        u64 cs_cycle_last;      /* last cycle value */
        u64 cs_mask;            /* clocksource mask */

diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index 83cb49ef2ab76a2f7b8fab18aae1cfb91a6fe493..d3c2a82a588b626ade23404f80eb9e83ed2df88b 100644 (file)
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -259,14 +259,14 @@ void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
 
 static void vdso_write_begin(struct vdso_data *vdata)
 {
-       ++vdso_data->seq_count;
+       ++vdso_data->tb_seq_count;
        smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */
 }
 
 static void vdso_write_end(struct vdso_data *vdata)
 {
        smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */
-       ++vdso_data->seq_count;
+       ++vdso_data->tb_seq_count;
 }
 
 static bool tk_is_cntvct(const struct timekeeper *tk)
@@ -290,10 +290,10 @@ static bool tk_is_cntvct(const struct timekeeper *tk)
  * counter again, making it even, indicating to userspace that the
  * update is finished.
  *
- * Userspace is expected to sample seq_count before reading any other
- * fields from the data page.  If seq_count is odd, userspace is
+ * Userspace is expected to sample tb_seq_count before reading any other
+ * fields from the data page.  If tb_seq_count is odd, userspace is
  * expected to wait until it becomes even.  After copying data from
- * the page, userspace must sample seq_count again; if it has changed
+ * the page, userspace must sample tb_seq_count again; if it has changed
  * from its previous value, userspace must retry the whole sequence.
  *
  * Calls to update_vsyscall are serialized by the timekeeping core.
@@ -311,18 +311,19 @@ void update_vsyscall(struct timekeeper *tk)
 
        vdso_write_begin(vdso_data);
 
-       vdso_data->tk_is_cntvct                 = tk_is_cntvct(tk);
+       vdso_data->use_syscall                  = !tk_is_cntvct(tk);
        vdso_data->xtime_coarse_sec             = tk->xtime_sec;
        vdso_data->xtime_coarse_nsec            = (u32)(tk->tkr_mono.xtime_nsec >>
                                                        tk->tkr_mono.shift);
        vdso_data->wtm_clock_sec                = wtm->tv_sec;
        vdso_data->wtm_clock_nsec               = wtm->tv_nsec;
 
-       if (vdso_data->tk_is_cntvct) {
+       if (!vdso_data->use_syscall) {
                vdso_data->cs_cycle_last        = tk->tkr_mono.cycle_last;
                vdso_data->xtime_clock_sec      = tk->xtime_sec;
                vdso_data->xtime_clock_snsec    = tk->tkr_mono.xtime_nsec;
-               vdso_data->cs_mult              = tk->tkr_mono.mult;
+               vdso_data->cs_mono_mult         = tk->tkr_mono.mult;
+               /* tkr_mono.shift == tkr_raw.shift */
                vdso_data->cs_shift             = tk->tkr_mono.shift;
                vdso_data->cs_mask              = tk->tkr_mono.mask;
        }

diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index 79214d5ff097044dc59e0394c87749e68d8f4d92..8cf13af1323cde6d4f6e31c4dbf53dfe28e8aee8 100644 (file)
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -1,18 +1,25 @@
 /*
- * Copyright 2015 Mentor Graphics Corporation.
+ * Userspace implementations of gettimeofday() and friends.
  *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; version 2 of the
- * License.
+ * Copyright (C) 2017 Cavium, Inc.
+ * Copyright (C) 2015 Mentor Graphics Corporation
+ * Copyright (C) 2012 ARM Limited
  *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ * Rewriten from arch64 version into C by: Andrew Pinski <apinski@cavium.com>
+ * Reworked and rebased over arm version by: Mark Salyzyn <salyzyn@android.com>
  */
 
 #include <linux/compiler.h>
@@ -31,32 +38,30 @@
 
 extern struct vdso_data *__get_datapage(void);
 
-static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)
-{
-       u32 seq;
-repeat:
-       seq = ACCESS_ONCE(vdata->seq_count);
-       if (seq & 1) {
-               cpu_relax();
-               goto repeat;
-       }
-       return seq;
-}
-
 static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
 {
        u32 seq;
 
-       seq = __vdso_read_begin(vdata);
+       do {
+               seq = READ_ONCE(vdata->tb_seq_count);
+
+               if ((seq & 1) == 0)
+                       break;
 
-       smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */
+               cpu_relax();
+       } while (true);
+
+       smp_rmb(); /* Pairs with second smp_wmb in update_vsyscall */
        return seq;
 }
 
 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
 {
-       smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */
-       return vdata->seq_count != start;
+       u32 seq;
+
+       smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */
+       seq = READ_ONCE(vdata->tb_seq_count);
+       return seq != start;
 }
 
 static notrace long clock_gettime_fallback(clockid_t _clkid,
@@ -127,7 +132,7 @@ static notrace u64 get_ns(struct vdso_data *vdata)
 
        cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;
 
-       nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;
+       nsec = (cycle_delta * vdata->cs_mono_mult) + vdata->xtime_clock_snsec;
        nsec >>= vdata->cs_shift;
 
        return nsec;
@@ -141,7 +146,7 @@ static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
        do {
                seq = vdso_read_begin(vdata);
 
-               if (!vdata->tk_is_cntvct)
+               if (vdata->use_syscall)
                        return -1;
 
                ts->tv_sec = vdata->xtime_clock_sec;
@@ -164,7 +169,7 @@ static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
        do {
                seq = vdso_read_begin(vdata);
 
-               if (!vdata->tk_is_cntvct)
+               if (vdata->use_syscall)
                        return -1;
 
                ts->tv_sec = vdata->xtime_clock_sec;