Commit | Line | Data |
---|---|---|
3d083395 SR |
1 | /* |
2 | * Code for replacing ftrace calls with jumps. | |
3 | * | |
4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
5 | * | |
6 | * Thanks goes to Ingo Molnar, for suggesting the idea. | |
7 | * Mathieu Desnoyers, for suggesting postponing the modifications. | |
8 | * Arjan van de Ven, for keeping me straight, and explaining to me | |
9 | * the dangers of modifying code on the run. | |
10 | */ | |
11 | ||
12 | #include <linux/spinlock.h> | |
13 | #include <linux/hardirq.h> | |
14 | #include <linux/ftrace.h> | |
15 | #include <linux/percpu.h> | |
16 | #include <linux/init.h> | |
17 | #include <linux/list.h> | |
18 | ||
19 | #define CALL_BACK 5 | |
20 | ||
21 | #define JMPFWD 0x03eb | |
22 | ||
23 | static unsigned short ftrace_jmp = JMPFWD; | |
24 | ||
25 | struct ftrace_record { | |
26 | struct dyn_ftrace rec; | |
27 | int failed; | |
28 | } __attribute__((packed)); | |
29 | ||
30 | struct ftrace_page { | |
31 | struct ftrace_page *next; | |
32 | int index; | |
33 | struct ftrace_record records[]; | |
34 | } __attribute__((packed)); | |
35 | ||
36 | #define ENTRIES_PER_PAGE \ | |
37 | ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct ftrace_record)) | |
38 | ||
39 | /* estimate from running different kernels */ | |
40 | #define NR_TO_INIT 10000 | |
41 | ||
42 | #define MCOUNT_ADDR ((long)(&mcount)) | |
43 | ||
44 | union ftrace_code_union { | |
45 | char code[5]; | |
46 | struct { | |
47 | char e8; | |
48 | int offset; | |
49 | } __attribute__((packed)); | |
50 | }; | |
51 | ||
52 | static struct ftrace_page *ftrace_pages_start; | |
53 | static struct ftrace_page *ftrace_pages; | |
54 | ||
55 | notrace struct dyn_ftrace *ftrace_alloc_shutdown_node(unsigned long ip) | |
56 | { | |
57 | struct ftrace_record *rec; | |
58 | unsigned short save; | |
59 | ||
60 | ip -= CALL_BACK; | |
61 | save = *(short *)ip; | |
62 | ||
63 | /* If this was already converted, skip it */ | |
64 | if (save == JMPFWD) | |
65 | return NULL; | |
66 | ||
67 | if (ftrace_pages->index == ENTRIES_PER_PAGE) { | |
68 | if (!ftrace_pages->next) | |
69 | return NULL; | |
70 | ftrace_pages = ftrace_pages->next; | |
71 | } | |
72 | ||
73 | rec = &ftrace_pages->records[ftrace_pages->index++]; | |
74 | ||
75 | return &rec->rec; | |
76 | } | |
77 | ||
78 | static int notrace | |
79 | ftrace_modify_code(unsigned long ip, unsigned char *old_code, | |
80 | unsigned char *new_code) | |
81 | { | |
82 | unsigned short old = *(unsigned short *)old_code; | |
83 | unsigned short new = *(unsigned short *)new_code; | |
84 | unsigned short replaced; | |
85 | int faulted = 0; | |
86 | ||
87 | /* | |
88 | * Note: Due to modules and __init, code can | |
89 | * disappear and change, we need to protect against faulting | |
90 | * as well as code changing. | |
91 | * | |
92 | * No real locking needed, this code is run through | |
93 | * kstop_machine. | |
94 | */ | |
95 | asm volatile ( | |
96 | "1: lock\n" | |
97 | " cmpxchg %w3, (%2)\n" | |
98 | "2:\n" | |
99 | ".section .fixup, \"ax\"\n" | |
100 | " movl $1, %0\n" | |
101 | "3: jmp 2b\n" | |
102 | ".previous\n" | |
103 | _ASM_EXTABLE(1b, 3b) | |
104 | : "=r"(faulted), "=a"(replaced) | |
105 | : "r"(ip), "r"(new), "0"(faulted), "a"(old) | |
106 | : "memory"); | |
107 | sync_core(); | |
108 | ||
109 | if (replaced != old) | |
110 | faulted = 2; | |
111 | ||
112 | return faulted; | |
113 | } | |
114 | ||
115 | static int notrace ftrace_calc_offset(long ip) | |
116 | { | |
117 | return (int)(MCOUNT_ADDR - ip); | |
118 | } | |
119 | ||
120 | notrace void ftrace_code_disable(struct dyn_ftrace *rec) | |
121 | { | |
122 | unsigned long ip; | |
123 | union ftrace_code_union save; | |
124 | struct ftrace_record *r = | |
125 | container_of(rec, struct ftrace_record, rec); | |
126 | ||
127 | ip = rec->ip; | |
128 | ||
129 | save.e8 = 0xe8; | |
130 | save.offset = ftrace_calc_offset(ip); | |
131 | ||
132 | /* move the IP back to the start of the call */ | |
133 | ip -= CALL_BACK; | |
134 | ||
135 | r->failed = ftrace_modify_code(ip, save.code, (char *)&ftrace_jmp); | |
136 | } | |
137 | ||
138 | static void notrace ftrace_replace_code(int saved) | |
139 | { | |
140 | unsigned char *new = NULL, *old = NULL; | |
141 | struct ftrace_record *rec; | |
142 | struct ftrace_page *pg; | |
143 | unsigned long ip; | |
144 | int i; | |
145 | ||
146 | if (saved) | |
147 | old = (char *)&ftrace_jmp; | |
148 | else | |
149 | new = (char *)&ftrace_jmp; | |
150 | ||
151 | for (pg = ftrace_pages_start; pg; pg = pg->next) { | |
152 | for (i = 0; i < pg->index; i++) { | |
153 | union ftrace_code_union calc; | |
154 | rec = &pg->records[i]; | |
155 | ||
156 | /* don't modify code that has already faulted */ | |
157 | if (rec->failed) | |
158 | continue; | |
159 | ||
160 | ip = rec->rec.ip; | |
161 | ||
162 | calc.e8 = 0xe8; | |
163 | calc.offset = ftrace_calc_offset(ip); | |
164 | ||
165 | if (saved) | |
166 | new = calc.code; | |
167 | else | |
168 | old = calc.code; | |
169 | ||
170 | ip -= CALL_BACK; | |
171 | ||
172 | rec->failed = ftrace_modify_code(ip, old, new); | |
173 | } | |
174 | } | |
175 | ||
176 | } | |
177 | ||
178 | notrace void ftrace_startup_code(void) | |
179 | { | |
180 | ftrace_replace_code(1); | |
181 | } | |
182 | ||
183 | notrace void ftrace_shutdown_code(void) | |
184 | { | |
185 | ftrace_replace_code(0); | |
186 | } | |
187 | ||
188 | notrace void ftrace_shutdown_replenish(void) | |
189 | { | |
190 | if (ftrace_pages->next) | |
191 | return; | |
192 | ||
193 | /* allocate another page */ | |
194 | ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL); | |
195 | } | |
196 | ||
197 | notrace int ftrace_shutdown_arch_init(void) | |
198 | { | |
199 | struct ftrace_page *pg; | |
200 | int cnt; | |
201 | int i; | |
202 | ||
203 | /* allocate a few pages */ | |
204 | ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL); | |
205 | if (!ftrace_pages_start) | |
206 | return -1; | |
207 | ||
208 | /* | |
209 | * Allocate a few more pages. | |
210 | * | |
211 | * TODO: have some parser search vmlinux before | |
212 | * final linking to find all calls to ftrace. | |
213 | * Then we can: | |
214 | * a) know how many pages to allocate. | |
215 | * and/or | |
216 | * b) set up the table then. | |
217 | * | |
218 | * The dynamic code is still necessary for | |
219 | * modules. | |
220 | */ | |
221 | ||
222 | pg = ftrace_pages = ftrace_pages_start; | |
223 | ||
224 | cnt = NR_TO_INIT / ENTRIES_PER_PAGE; | |
225 | ||
226 | for (i = 0; i < cnt; i++) { | |
227 | pg->next = (void *)get_zeroed_page(GFP_KERNEL); | |
228 | ||
229 | /* If we fail, we'll try later anyway */ | |
230 | if (!pg->next) | |
231 | break; | |
232 | ||
233 | pg = pg->next; | |
234 | } | |
235 | ||
236 | return 0; | |
237 | } |