kernel/time/sched_clock.c

   1 /*
   2  * sched_clock.c: support for extending counters to full 64-bit ns counter
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8 #include <linux/clocksource.h>
   9 #include <linux/init.h>
  10 #include <linux/jiffies.h>
  11 #include <linux/ktime.h>
  12 #include <linux/kernel.h>
  13 #include <linux/moduleparam.h>
  14 #include <linux/sched.h>
  15 #include <linux/syscore_ops.h>
  16 #include <linux/hrtimer.h>
  17 #include <linux/sched_clock.h>
  18 #include <linux/seqlock.h>
  19 #include <linux/bitops.h>
  20
  21 struct clock_data {
  22         ktime_t wrap_kt;
  23         u64 epoch_ns;
  24         u64 epoch_cyc;
  25         u64 epoch_cyc_copy; /*add this to protect read & write share data (cd.epoch_ns/cd.epoch_cyc) sync (same as arch/arm/sched_clock.c)*/
  26         seqcount_t seq;
  27         unsigned long rate;
  28         u32 mult;
  29         u32 shift;
  30         bool suspended;
  31 };
  32
  33 static struct hrtimer sched_clock_timer;
  34 static int irqtime = -1;
  35
  36 core_param(irqtime, irqtime, int, 0400);
  37
  38 static struct clock_data cd = {
  39         .mult   = NSEC_PER_SEC / HZ,
  40 };
  41
  42 static u64 __read_mostly sched_clock_mask;
  43
  44 static u64 notrace jiffy_sched_clock_read(void)
  45 {
  46         /*
  47          * We don't need to use get_jiffies_64 on 32-bit arches here
  48          * because we register with BITS_PER_LONG
  49          */
  50         return (u64)(jiffies - INITIAL_JIFFIES);
  51 }
  52
  53 static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read;
  54
  55 static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift)
  56 {
  57         return (cyc * mult) >> shift;
  58 }
  59
  60 unsigned long long notrace sched_clock(void)
  61 {
  62         u64 epoch_ns;
  63         u64 epoch_cyc;
  64         u64 cyc;
  65         unsigned long seq;
  66
  67         if (cd.suspended)
  68                 return cd.epoch_ns;
  69     /*
  70        * used cd.epoch_cyc_copy instead of read/write seqcount
  71        * because of read/write seqcount cannot sync the read/write flow
  72        * race condition will happen for epoch_cyc/epoch_ns
  73        */
  74         do {
  75                 //seq = raw_read_seqcount_begin(&cd.seq);
  76                 //seq = read_seqcount_begin(&cd.seq);
  77                 epoch_cyc = cd.epoch_cyc;
  78                 smp_rmb();
  79                 epoch_ns = cd.epoch_ns;
  80                 smp_rmb();
  81         } while (epoch_cyc != cd.epoch_cyc_copy);
  82
  83         cyc = read_sched_clock();
  84         cyc = (cyc - epoch_cyc) & sched_clock_mask;
  85         return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift);
  86 }
  87
  88 /*
  89  * Atomically update the sched_clock epoch.
  90  */
  91 static void notrace update_sched_clock(void)
  92 {
  93         unsigned long flags;
  94         u64 cyc;
  95         u64 ns;
  96
  97         cyc = read_sched_clock();
  98         ns = cd.epoch_ns +
  99                 cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
 100                           cd.mult, cd.shift);
 101
 102         raw_local_irq_save(flags);
 103         /*
 104         *  same with sched_clock() comment
 105         */
 106
 107         //raw_write_seqcount_begin(&cd.seq);
 108         //write_seqcount_begin(&cd.seq);
 109         cd.epoch_cyc_copy = cyc;
 110         smp_wmb();
 111         cd.epoch_ns = ns;
 112         smp_wmb();
 113         cd.epoch_cyc = cyc;
 114         //raw_write_seqcount_end(&cd.seq);
 115         //write_seqcount_end(&cd.seq);
 116         raw_local_irq_restore(flags);
 117 }
 118
 119 static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
 120 {
 121         update_sched_clock();
 122         hrtimer_forward_now(hrt, cd.wrap_kt);
 123         return HRTIMER_RESTART;
 124 }
 125
 126 void __init sched_clock_register(u64 (*read)(void), int bits,
 127                                  unsigned long rate)
 128 {
 129         u64 res, wrap, new_mask, new_epoch, cyc, ns;
 130         u32 new_mult, new_shift;
 131         ktime_t new_wrap_kt;
 132         unsigned long r;
 133         char r_unit;
 134
 135         if (cd.rate > rate)
 136                 return;
 137
 138         WARN_ON(!irqs_disabled());
 139
 140         /* calculate the mult/shift to convert counter ticks to ns. */
 141         clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 0);
 142
 143         //new_mask = CLOCKSOURCE_MASK(bits);
 144         /* calculate how many ns until we wrap */
 145         //wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask);
 146
 147         //under kernel 3.10
 148         new_mask = (1ULL<<(bits))-1;
 149         wrap = cyc_to_ns((1ULL << bits) - 1, new_mult, new_shift);
 150
 151         new_wrap_kt = ns_to_ktime(wrap - (wrap >> 3));
 152
 153         /* update epoch for new counter and update epoch_ns from old counter*/
 154         new_epoch = read();
 155         cyc = read_sched_clock();
 156         ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask,
 157                           cd.mult, cd.shift);
 158
 159         //raw_write_seqcount_begin(&cd.seq);
 160         write_seqcount_begin(&cd.seq);
 161         read_sched_clock = read;
 162         sched_clock_mask = new_mask;
 163         cd.rate = rate;
 164         cd.wrap_kt = new_wrap_kt;
 165         cd.mult = new_mult;
 166         cd.shift = new_shift;
 167         cd.epoch_cyc = new_epoch;
 168         cd.epoch_ns = ns;
 169         //raw_write_seqcount_end(&cd.seq);
 170         write_seqcount_end(&cd.seq);
 171
 172         r = rate;
 173         if (r >= 4000000) {
 174                 r /= 1000000;
 175                 r_unit = 'M';
 176         } else if (r >= 1000) {
 177                 r /= 1000;
 178                 r_unit = 'k';
 179         } else
 180                 r_unit = ' ';
 181
 182         /* calculate the ns resolution of this counter */
 183         res = cyc_to_ns(1ULL, new_mult, new_shift);
 184
 185         pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n",
 186                 bits, r, r_unit, res, wrap);
 187
 188         /* Enable IRQ time accounting if we have a fast enough sched_clock */
 189         if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
 190                 enable_sched_clock_irqtime();
 191
 192         pr_debug("Registered %pF as sched_clock source\n", read);
 193 }
 194
 195 void __init sched_clock_postinit(void)
 196 {
 197         /*
 198          * If no sched_clock function has been provided at that point,
 199          * make it the final one one.
 200          */
 201         if (read_sched_clock == jiffy_sched_clock_read)
 202                 sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
 203
 204         update_sched_clock();
 205
 206         /*
 207          * Start the timer to keep sched_clock() properly updated and
 208          * sets the initial epoch.
 209          */
 210         hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 211         sched_clock_timer.function = sched_clock_poll;
 212         hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
 213 }
 214
 215 static int sched_clock_suspend(void)
 216 {
 217         update_sched_clock();
 218         hrtimer_cancel(&sched_clock_timer);
 219         cd.suspended = true;
 220         return 0;
 221 }
 222
 223 static void sched_clock_resume(void)
 224 {
 225         cd.epoch_cyc = read_sched_clock();
 226         cd.epoch_cyc_copy = cd.epoch_cyc;
 227         hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
 228         cd.suspended = false;
 229 }
 230
 231 static struct syscore_ops sched_clock_ops = {
 232         .suspend = sched_clock_suspend,
 233         .resume = sched_clock_resume,
 234 };
 235
 236 static int __init sched_clock_syscore_init(void)
 237 {
 238         register_syscore_ops(&sched_clock_ops);
 239         return 0;
 240 }
 241 device_initcall(sched_clock_syscore_init);