arch/x86_64/kernel/vsyscall.c

   1 /*
   2  *  linux/arch/x86_64/kernel/vsyscall.c
   3  *
   4  *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
   5  *  Copyright 2003 Andi Kleen, SuSE Labs.
   6  *
   7  *  Thanks to hpa@transmeta.com for some useful hint.
   8  *  Special thanks to Ingo Molnar for his early experience with
   9  *  a different vsyscall implementation for Linux/IA32 and for the name.
  10  *
  11  *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
  12  *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
  13  *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
  14  *  jumping out of line if necessary. We cannot add more with this
  15  *  mechanism because older kernels won't return -ENOSYS.
  16  *  If we want more than four we need a vDSO.
  17  *
  18  *  Note: the concept clashes with user mode linux. If you use UML and
  19  *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
  20  */
  21
  22 #include <linux/time.h>
  23 #include <linux/init.h>
  24 #include <linux/kernel.h>
  25 #include <linux/timer.h>
  26 #include <linux/seqlock.h>
  27 #include <linux/jiffies.h>
  28 #include <linux/sysctl.h>
  29 #include <linux/clocksource.h>
  30 #include <linux/getcpu.h>
  31 #include <linux/cpu.h>
  32 #include <linux/smp.h>
  33 #include <linux/notifier.h>
  34
  35 #include <asm/vsyscall.h>
  36 #include <asm/pgtable.h>
  37 #include <asm/page.h>
  38 #include <asm/unistd.h>
  39 #include <asm/fixmap.h>
  40 #include <asm/errno.h>
  41 #include <asm/io.h>
  42 #include <asm/segment.h>
  43 #include <asm/desc.h>
  44 #include <asm/topology.h>
  45
  46 #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
  47 #define __syscall_clobber "r11","rcx","memory"
  48
  49 struct vsyscall_gtod_data_t {
  50         seqlock_t lock;
  51         int sysctl_enabled;
  52         struct timeval wall_time_tv;
  53         struct timezone sys_tz;
  54         cycle_t offset_base;
  55         struct clocksource clock;
  56 };
  57 int __vgetcpu_mode __section_vgetcpu_mode;
  58
  59 struct vsyscall_gtod_data_t __vsyscall_gtod_data __section_vsyscall_gtod_data =
  60 {
  61         .lock = SEQLOCK_UNLOCKED,
  62         .sysctl_enabled = 1,
  63 };
  64
  65 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
  66 {
  67         unsigned long flags;
  68
  69         write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  70         /* copy vsyscall data */
  71         vsyscall_gtod_data.clock = *clock;
  72         vsyscall_gtod_data.wall_time_tv.tv_sec = wall_time->tv_sec;
  73         vsyscall_gtod_data.wall_time_tv.tv_usec = wall_time->tv_nsec/1000;
  74         vsyscall_gtod_data.sys_tz = sys_tz;
  75         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  76 }
  77
  78 /* RED-PEN may want to readd seq locking, but then the variable should be
  79  * write-once.
  80  */
  81 static __always_inline void do_get_tz(struct timezone * tz)
  82 {
  83         *tz = __vsyscall_gtod_data.sys_tz;
  84 }
  85
  86 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
  87 {
  88         int ret;
  89         asm volatile("vsysc2: syscall"
  90                 : "=a" (ret)
  91                 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
  92                 : __syscall_clobber );
  93         return ret;
  94 }
  95
  96 static __always_inline long time_syscall(long *t)
  97 {
  98         long secs;
  99         asm volatile("vsysc1: syscall"
 100                 : "=a" (secs)
 101                 : "0" (__NR_time),"D" (t) : __syscall_clobber);
 102         return secs;
 103 }
 104
 105 static __always_inline void do_vgettimeofday(struct timeval * tv)
 106 {
 107         cycle_t now, base, mask, cycle_delta;
 108         unsigned long seq, mult, shift, nsec_delta;
 109         cycle_t (*vread)(void);
 110         do {
 111                 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
 112
 113                 vread = __vsyscall_gtod_data.clock.vread;
 114                 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
 115                         gettimeofday(tv,NULL);
 116                         return;
 117                 }
 118                 now = vread();
 119                 base = __vsyscall_gtod_data.clock.cycle_last;
 120                 mask = __vsyscall_gtod_data.clock.mask;
 121                 mult = __vsyscall_gtod_data.clock.mult;
 122                 shift = __vsyscall_gtod_data.clock.shift;
 123
 124                 *tv = __vsyscall_gtod_data.wall_time_tv;
 125
 126         } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
 127
 128         /* calculate interval: */
 129         cycle_delta = (now - base) & mask;
 130         /* convert to nsecs: */
 131         nsec_delta = (cycle_delta * mult) >> shift;
 132
 133         /* convert to usecs and add to timespec: */
 134         tv->tv_usec += nsec_delta / NSEC_PER_USEC;
 135         while (tv->tv_usec > USEC_PER_SEC) {
 136                 tv->tv_sec += 1;
 137                 tv->tv_usec -= USEC_PER_SEC;
 138         }
 139 }
 140
 141 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
 142 {
 143         if (tv)
 144                 do_vgettimeofday(tv);
 145         if (tz)
 146                 do_get_tz(tz);
 147         return 0;
 148 }
 149
 150 /* This will break when the xtime seconds get inaccurate, but that is
 151  * unlikely */
 152 time_t __vsyscall(1) vtime(time_t *t)
 153 {
 154         if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
 155                 return time_syscall(t);
 156         else if (t)
 157                 *t = __vsyscall_gtod_data.wall_time_tv.tv_sec;
 158         return __vsyscall_gtod_data.wall_time_tv.tv_sec;
 159 }
 160
 161 /* Fast way to get current CPU and node.
 162    This helps to do per node and per CPU caches in user space.
 163    The result is not guaranteed without CPU affinity, but usually
 164    works out because the scheduler tries to keep a thread on the same
 165    CPU.
 166
 167    tcache must point to a two element sized long array.
 168    All arguments can be NULL. */
 169 long __vsyscall(2)
 170 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 171 {
 172         unsigned int dummy, p;
 173         unsigned long j = 0;
 174
 175         /* Fast cache - only recompute value once per jiffies and avoid
 176            relatively costly rdtscp/cpuid otherwise.
 177            This works because the scheduler usually keeps the process
 178            on the same CPU and this syscall doesn't guarantee its
 179            results anyways.
 180            We do this here because otherwise user space would do it on
 181            its own in a likely inferior way (no access to jiffies).
 182            If you don't like it pass NULL. */
 183         if (tcache && tcache->blob[0] == (j = __jiffies)) {
 184                 p = tcache->blob[1];
 185         } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 186                 /* Load per CPU data from RDTSCP */
 187                 rdtscp(dummy, dummy, p);
 188         } else {
 189                 /* Load per CPU data from GDT */
 190                 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
 191         }
 192         if (tcache) {
 193                 tcache->blob[0] = j;
 194                 tcache->blob[1] = p;
 195         }
 196         if (cpu)
 197                 *cpu = p & 0xfff;
 198         if (node)
 199                 *node = p >> 12;
 200         return 0;
 201 }
 202
 203 long __vsyscall(3) venosys_1(void)
 204 {
 205         return -ENOSYS;
 206 }
 207
 208 #ifdef CONFIG_SYSCTL
 209
 210 #define SYSCALL 0x050f
 211 #define NOP2    0x9090
 212
 213 /*
 214  * NOP out syscall in vsyscall page when not needed.
 215  */
 216 static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
 217                         void __user *buffer, size_t *lenp, loff_t *ppos)
 218 {
 219         extern u16 vsysc1, vsysc2;
 220         u16 __iomem *map1;
 221         u16 __iomem *map2;
 222         int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
 223         if (!write)
 224                 return ret;
 225         /* gcc has some trouble with __va(__pa()), so just do it this
 226            way. */
 227         map1 = ioremap(__pa_symbol(&vsysc1), 2);
 228         if (!map1)
 229                 return -ENOMEM;
 230         map2 = ioremap(__pa_symbol(&vsysc2), 2);
 231         if (!map2) {
 232                 ret = -ENOMEM;
 233                 goto out;
 234         }
 235         if (!vsyscall_gtod_data.sysctl_enabled) {
 236                 writew(SYSCALL, map1);
 237                 writew(SYSCALL, map2);
 238         } else {
 239                 writew(NOP2, map1);
 240                 writew(NOP2, map2);
 241         }
 242         iounmap(map2);
 243 out:
 244         iounmap(map1);
 245         return ret;
 246 }
 247
 248 static int vsyscall_sysctl_nostrat(ctl_table *t, int __user *name, int nlen,
 249                                 void __user *oldval, size_t __user *oldlenp,
 250                                 void __user *newval, size_t newlen)
 251 {
 252         return -ENOSYS;
 253 }
 254
 255 static ctl_table kernel_table2[] = {
 256         { .ctl_name = 99, .procname = "vsyscall64",
 257           .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
 258           .mode = 0644,
 259           .strategy = vsyscall_sysctl_nostrat,
 260           .proc_handler = vsyscall_sysctl_change },
 261         {}
 262 };
 263
 264 static ctl_table kernel_root_table2[] = {
 265         { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
 266           .child = kernel_table2 },
 267         {}
 268 };
 269
 270 #endif
 271
 272 /* Assume __initcall executes before all user space. Hopefully kmod
 273    doesn't violate that. We'll find out if it does. */
 274 static void __cpuinit vsyscall_set_cpu(int cpu)
 275 {
 276         unsigned long *d;
 277         unsigned long node = 0;
 278 #ifdef CONFIG_NUMA
 279         node = cpu_to_node[cpu];
 280 #endif
 281         if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
 282                 write_rdtscp_aux((node << 12) | cpu);
 283
 284         /* Store cpu number in limit so that it can be loaded quickly
 285            in user space in vgetcpu.
 286            12 bits for the CPU and 8 bits for the node. */
 287         d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
 288         *d = 0x0f40000000000ULL;
 289         *d |= cpu;
 290         *d |= (node & 0xf) << 12;
 291         *d |= (node >> 4) << 48;
 292 }
 293
 294 static void __cpuinit cpu_vsyscall_init(void *arg)
 295 {
 296         /* preemption should be already off */
 297         vsyscall_set_cpu(raw_smp_processor_id());
 298 }
 299
 300 static int __cpuinit
 301 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
 302 {
 303         long cpu = (long)arg;
 304         if (action == CPU_ONLINE)
 305                 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
 306         return NOTIFY_DONE;
 307 }
 308
 309 static void __init map_vsyscall(void)
 310 {
 311         extern char __vsyscall_0;
 312         unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
 313
 314         /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
 315         __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
 316 }
 317
 318 static int __init vsyscall_init(void)
 319 {
 320         BUG_ON(((unsigned long) &vgettimeofday !=
 321                         VSYSCALL_ADDR(__NR_vgettimeofday)));
 322         BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
 323         BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
 324         BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
 325         map_vsyscall();
 326 #ifdef CONFIG_SYSCTL
 327         register_sysctl_table(kernel_root_table2);
 328 #endif
 329         on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
 330         hotcpu_notifier(cpu_vsyscall_notifier, 0);
 331         return 0;
 332 }
 333
 334 __initcall(vsyscall_init);