projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] x86_64: Fix a small missing schedule race
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / x86_64 / kernel / entry.S
CommitLineData
1da177e4
LT		 1/*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 *
8 * $Id$
9 */
10
11/*
12 * entry.S contains the system-call and fault low-level handling routines.
13 *
14 * NOTE: This code handles signal-recognition, which happens every time
15 * after an interrupt and after each system call.
16 *
17 * Normal syscalls and interrupts don't save a full stack frame, this is
18 * only done for syscall tracing, signals or fork/exec et.al.
19 *
20 * A note on terminology:
21 * - top of stack: Architecture defined interrupt frame from SS to RIP
22 * at the top of the kernel process stack.
23 * - partial stack frame: partially saved registers upto R11.
24 * - full stack frame: Like partial stack frame, but all register saved.
25 *
26 * TODO:
27 * - schedule it carefully for the final hardware.
28 */
29
30#define ASSEMBLY 1
31#include <linux/config.h>
32#include <linux/linkage.h>
33#include <asm/segment.h>
34#include <asm/smp.h>
35#include <asm/cache.h>
36#include <asm/errno.h>
37#include <asm/dwarf2.h>
38#include <asm/calling.h>
39#include <asm/offset.h>
40#include <asm/msr.h>
41#include <asm/unistd.h>
42#include <asm/thread_info.h>
43#include <asm/hw_irq.h>
44
45 .code64
46
47#ifdef CONFIG_PREEMPT
48#define preempt_stop cli
49#else
50#define preempt_stop
51#define retint_kernel retint_restore_args
52#endif
53
54/*
55 * C code is not supposed to know about undefined top of stack. Every time
56 * a C function with an pt_regs argument is called from the SYSCALL based
57 * fast path FIXUP_TOP_OF_STACK is needed.
58 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
59 * manipulation.
60 */
61
62 /* %rsp:at FRAMEEND */
63 .macro FIXUP_TOP_OF_STACK tmp
64 movq %gs:pda_oldrsp,\tmp
65 movq \tmp,RSP(%rsp)
66 movq $__USER_DS,SS(%rsp)
67 movq $__USER_CS,CS(%rsp)
68 movq $-1,RCX(%rsp)
69 movq R11(%rsp),\tmp /* get eflags */
70 movq \tmp,EFLAGS(%rsp)
71 .endm
72
73 .macro RESTORE_TOP_OF_STACK tmp,offset=0
74 movq RSP-\offset(%rsp),\tmp
75 movq \tmp,%gs:pda_oldrsp
76 movq EFLAGS-\offset(%rsp),\tmp
77 movq \tmp,R11-\offset(%rsp)
78 .endm
79
80 .macro FAKE_STACK_FRAME child_rip
81 /* push in order ss, rsp, eflags, cs, rip */
82 xorq %rax, %rax
83 pushq %rax /* ss */
84 CFI_ADJUST_CFA_OFFSET 8
85 pushq %rax /* rsp */
86 CFI_ADJUST_CFA_OFFSET 8
87 CFI_OFFSET rip,0
88 pushq $(1<<9) /* eflags - interrupts on */
89 CFI_ADJUST_CFA_OFFSET 8
90 pushq $__KERNEL_CS /* cs */
91 CFI_ADJUST_CFA_OFFSET 8
92 pushq \child_rip /* rip */
93 CFI_ADJUST_CFA_OFFSET 8
94 CFI_OFFSET rip,0
95 pushq %rax /* orig rax */
96 CFI_ADJUST_CFA_OFFSET 8
97 .endm
98
99 .macro UNFAKE_STACK_FRAME
100 addq $8*6, %rsp
101 CFI_ADJUST_CFA_OFFSET -(6*8)
102 .endm
103
104 .macro CFI_DEFAULT_STACK
105 CFI_ADJUST_CFA_OFFSET (SS)
106 CFI_OFFSET r15,R15-SS
107 CFI_OFFSET r14,R14-SS
108 CFI_OFFSET r13,R13-SS
109 CFI_OFFSET r12,R12-SS
110 CFI_OFFSET rbp,RBP-SS
111 CFI_OFFSET rbx,RBX-SS
112 CFI_OFFSET r11,R11-SS
113 CFI_OFFSET r10,R10-SS
114 CFI_OFFSET r9,R9-SS
115 CFI_OFFSET r8,R8-SS
116 CFI_OFFSET rax,RAX-SS
117 CFI_OFFSET rcx,RCX-SS
118 CFI_OFFSET rdx,RDX-SS
119 CFI_OFFSET rsi,RSI-SS
120 CFI_OFFSET rdi,RDI-SS
121 CFI_OFFSET rsp,RSP-SS
122 CFI_OFFSET rip,RIP-SS
123 .endm
124/*
125 * A newly forked process directly context switches into this.
126 */
127/* rdi: prev */
128ENTRY(ret_from_fork)
129 CFI_STARTPROC
130 CFI_DEFAULT_STACK
131 call schedule_tail
132 GET_THREAD_INFO(%rcx)
133 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
134 jnz rff_trace
135rff_action:
136 RESTORE_REST
137 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
138 je int_ret_from_sys_call
139 testl $_TIF_IA32,threadinfo_flags(%rcx)
140 jnz int_ret_from_sys_call
141 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
142 jmp ret_from_sys_call
143rff_trace:
144 movq %rsp,%rdi
145 call syscall_trace_leave
146 GET_THREAD_INFO(%rcx)
147 jmp rff_action
148 CFI_ENDPROC
149
150/*
151 * System call entry. Upto 6 arguments in registers are supported.
152 *
153 * SYSCALL does not save anything on the stack and does not change the
154 * stack pointer.
155 */
156
157/*
158 * Register setup:
159 * rax system call number
160 * rdi arg0
161 * rcx return address for syscall/sysret, C arg3
162 * rsi arg1
163 * rdx arg2
164 * r10 arg3 (--> moved to rcx for C)
165 * r8 arg4
166 * r9 arg5
167 * r11 eflags for syscall/sysret, temporary for C
168 * r12-r15,rbp,rbx saved by C code, not touched.
169 *
170 * Interrupts are off on entry.
171 * Only called from user space.
172 *
173 * XXX if we had a free scratch register we could save the RSP into the stack frame
174 * and report it properly in ps. Unfortunately we haven't.
175 */
176
177ENTRY(system_call)
178 CFI_STARTPROC
179 swapgs
180 movq %rsp,%gs:pda_oldrsp
181 movq %gs:pda_kernelstack,%rsp
182 sti
183 SAVE_ARGS 8,1
184 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
185 movq %rcx,RIP-ARGOFFSET(%rsp)
186 GET_THREAD_INFO(%rcx)
187 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
188 jnz tracesys
189 cmpq $__NR_syscall_max,%rax
190 ja badsys
191 movq %r10,%rcx
192 call *sys_call_table(,%rax,8) # XXX: rip relative
193 movq %rax,RAX-ARGOFFSET(%rsp)
194/*
195 * Syscall return path ending with SYSRET (fast path)
196 * Has incomplete stack frame and undefined top of stack.
197 */
198 .globl ret_from_sys_call
199ret_from_sys_call:
11b854b2 200 movl $_TIF_ALLWORK_MASK,%edi
1da177e4
LT		 201 /* edi: flagmask */
202sysret_check:
203 GET_THREAD_INFO(%rcx)
204 cli
205 movl threadinfo_flags(%rcx),%edx
206 andl %edi,%edx
207 jnz sysret_careful
208 movq RIP-ARGOFFSET(%rsp),%rcx
209 RESTORE_ARGS 0,-ARG_SKIP,1
210 movq %gs:pda_oldrsp,%rsp
211 swapgs
212 sysretq
213
214 /* Handle reschedules */
215 /* edx: work, edi: workmask */
216sysret_careful:
217 bt $TIF_NEED_RESCHED,%edx
218 jnc sysret_signal
219 sti
220 pushq %rdi
221 call schedule
222 popq %rdi
223 jmp sysret_check
224
225 /* Handle a signal */
11b854b2 226 /* edx: work flags (arg3) */
1da177e4
LT		 227sysret_signal:
228 sti
1da177e4
LT		 229 leaq do_notify_resume(%rip),%rax
230 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
231 xorl %esi,%esi # oldset -> arg2
232 call ptregscall_common
1da177e4
LT		 233 jmp sysret_check
234
235 /* Do syscall tracing */
236tracesys:
237 SAVE_REST
238 movq $-ENOSYS,RAX(%rsp)
239 FIXUP_TOP_OF_STACK %rdi
240 movq %rsp,%rdi
241 call syscall_trace_enter
242 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
243 RESTORE_REST
244 cmpq $__NR_syscall_max,%rax
245 ja 1f
246 movq %r10,%rcx /* fixup for C */
247 call *sys_call_table(,%rax,8)
248 movq %rax,RAX-ARGOFFSET(%rsp)
2491: SAVE_REST
250 movq %rsp,%rdi
251 call syscall_trace_leave
252 RESTORE_TOP_OF_STACK %rbx
253 RESTORE_REST
254 jmp ret_from_sys_call
255
256badsys:
257 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
258 jmp ret_from_sys_call
259
260/*
261 * Syscall return path ending with IRET.
262 * Has correct top of stack, but partial stack frame.
263 */
264ENTRY(int_ret_from_sys_call)
265 cli
266 testl $3,CS-ARGOFFSET(%rsp)
267 je retint_restore_args
268 movl $_TIF_ALLWORK_MASK,%edi
269 /* edi: mask to check */
270int_with_check:
271 GET_THREAD_INFO(%rcx)
272 movl threadinfo_flags(%rcx),%edx
273 andl %edi,%edx
274 jnz int_careful
275 jmp retint_swapgs
276
277 /* Either reschedule or signal or syscall exit tracking needed. */
278 /* First do a reschedule test. */
279 /* edx: work, edi: workmask */
280int_careful:
281 bt $TIF_NEED_RESCHED,%edx
282 jnc int_very_careful
283 sti
284 pushq %rdi
285 call schedule
286 popq %rdi
cdd219cd 287 cli
1da177e4
LT		 288 jmp int_with_check
289
290 /* handle signals and tracing -- both require a full stack frame */
291int_very_careful:
292 sti
293 SAVE_REST
294 /* Check for syscall exit trace */
295 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
296 jz int_signal
297 pushq %rdi
298 leaq 8(%rsp),%rdi # &ptregs -> arg1
299 call syscall_trace_leave
300 popq %rdi
36c1104e 301 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
1da177e4
LT		 302 jmp int_restore_rest
303
304int_signal:
305 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
306 jz 1f
307 movq %rsp,%rdi # &ptregs -> arg1
308 xorl %esi,%esi # oldset -> arg2
309 call do_notify_resume
3101: movl $_TIF_NEED_RESCHED,%edi
311int_restore_rest:
312 RESTORE_REST
313 jmp int_with_check
314 CFI_ENDPROC
315
316/*
317 * Certain special system calls that need to save a complete full stack frame.
318 */
319
320 .macro PTREGSCALL label,func,arg
321 .globl \label
322\label:
323 leaq \func(%rip),%rax
324 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
325 jmp ptregscall_common
326 .endm
327
328 PTREGSCALL stub_clone, sys_clone, %r8
329 PTREGSCALL stub_fork, sys_fork, %rdi
330 PTREGSCALL stub_vfork, sys_vfork, %rdi
331 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
332 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
333 PTREGSCALL stub_iopl, sys_iopl, %rsi
334
335ENTRY(ptregscall_common)
336 CFI_STARTPROC
337 popq %r11
338 CFI_ADJUST_CFA_OFFSET -8
339 SAVE_REST
340 movq %r11, %r15
341 FIXUP_TOP_OF_STACK %r11
342 call *%rax
343 RESTORE_TOP_OF_STACK %r11
344 movq %r15, %r11
345 RESTORE_REST
346 pushq %r11
347 CFI_ADJUST_CFA_OFFSET 8
348 ret
349 CFI_ENDPROC
350
351ENTRY(stub_execve)
352 CFI_STARTPROC
353 popq %r11
354 CFI_ADJUST_CFA_OFFSET -8
355 SAVE_REST
356 movq %r11, %r15
357 FIXUP_TOP_OF_STACK %r11
358 call sys_execve
359 GET_THREAD_INFO(%rcx)
360 bt $TIF_IA32,threadinfo_flags(%rcx)
361 jc exec_32bit
362 RESTORE_TOP_OF_STACK %r11
363 movq %r15, %r11
364 RESTORE_REST
365 push %r11
366 ret
367
368exec_32bit:
369 CFI_ADJUST_CFA_OFFSET REST_SKIP
370 movq %rax,RAX(%rsp)
371 RESTORE_REST
372 jmp int_ret_from_sys_call
373 CFI_ENDPROC
374
375/*
376 * sigreturn is special because it needs to restore all registers on return.
377 * This cannot be done with SYSRET, so use the IRET return path instead.
378 */
379ENTRY(stub_rt_sigreturn)
380 CFI_STARTPROC
381 addq $8, %rsp
382 SAVE_REST
383 movq %rsp,%rdi
384 FIXUP_TOP_OF_STACK %r11
385 call sys_rt_sigreturn
386 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
387 RESTORE_REST
388 jmp int_ret_from_sys_call
389 CFI_ENDPROC
390
391/*
392 * Interrupt entry/exit.
393 *
394 * Interrupt entry points save only callee clobbered registers in fast path.
395 *
396 * Entry runs with interrupts off.
397 */
398
399/* 0(%rsp): interrupt number */
400 .macro interrupt func
401 CFI_STARTPROC simple
402 CFI_DEF_CFA rsp,(SS-RDI)
403 CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
404 CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
405 cld
406#ifdef CONFIG_DEBUG_INFO
407 SAVE_ALL
408 movq %rsp,%rdi
409 /*
410 * Setup a stack frame pointer. This allows gdb to trace
411 * back to the original stack.
412 */
413 movq %rsp,%rbp
414 CFI_DEF_CFA_REGISTER rbp
415#else
416 SAVE_ARGS
417 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
418#endif
419 testl $3,CS(%rdi)
420 je 1f
421 swapgs
4221: addl $1,%gs:pda_irqcount # RED-PEN should check preempt count
423 movq %gs:pda_irqstackptr,%rax
424 cmoveq %rax,%rsp
425 pushq %rdi # save old stack
426 call \func
427 .endm
428
429ENTRY(common_interrupt)
430 interrupt do_IRQ
431 /* 0(%rsp): oldrsp-ARGOFFSET */
432ret_from_intr:
433 popq %rdi
434 cli
435 subl $1,%gs:pda_irqcount
436#ifdef CONFIG_DEBUG_INFO
437 movq RBP(%rdi),%rbp
438#endif
439 leaq ARGOFFSET(%rdi),%rsp
440exit_intr:
441 GET_THREAD_INFO(%rcx)
442 testl $3,CS-ARGOFFSET(%rsp)
443 je retint_kernel
444
445 /* Interrupt came from user space */
446 /*
447 * Has a correct top of stack, but a partial stack frame
448 * %rcx: thread info. Interrupts off.
449 */
450retint_with_reschedule:
451 movl $_TIF_WORK_MASK,%edi
452retint_check:
453 movl threadinfo_flags(%rcx),%edx
454 andl %edi,%edx
455 jnz retint_careful
456retint_swapgs:
1da177e4
LT		 457 swapgs
458retint_restore_args:
459 cli
460 RESTORE_ARGS 0,8,0
461iret_label:
462 iretq
463
464 .section __ex_table,"a"
465 .quad iret_label,bad_iret
466 .previous
467 .section .fixup,"ax"
468 /* force a signal here? this matches i386 behaviour */
469 /* running with kernel gs */
470bad_iret:
471 movq $-9999,%rdi /* better code? */
472 jmp do_exit
473 .previous
474
475 /* edi: workmask, edx: work */
476retint_careful:
477 bt $TIF_NEED_RESCHED,%edx
478 jnc retint_signal
479 sti
480 pushq %rdi
481 call schedule
482 popq %rdi
483 GET_THREAD_INFO(%rcx)
484 cli
485 jmp retint_check
486
487retint_signal:
1da177e4
LT		 488 sti
489 SAVE_REST
490 movq $-1,ORIG_RAX(%rsp)
491 xorq %rsi,%rsi # oldset
492 movq %rsp,%rdi # &pt_regs
493 call do_notify_resume
494 RESTORE_REST
495 cli
1da177e4
LT		 496 GET_THREAD_INFO(%rcx)
497 jmp retint_check
498
499#ifdef CONFIG_PREEMPT
500 /* Returning to kernel space. Check if we need preemption */
501 /* rcx: threadinfo. interrupts off. */
502 .p2align
503retint_kernel:
504 cmpl $0,threadinfo_preempt_count(%rcx)
505 jnz retint_restore_args
506 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
507 jnc retint_restore_args
508 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
509 jnc retint_restore_args
510 call preempt_schedule_irq
511 jmp exit_intr
512#endif
513 CFI_ENDPROC
514
515/*
516 * APIC interrupts.
517 */
518 .macro apicinterrupt num,func
519 pushq $\num-256
520 interrupt \func
521 jmp ret_from_intr
522 CFI_ENDPROC
523 .endm
524
525ENTRY(thermal_interrupt)
526 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
527
528#ifdef CONFIG_SMP
529ENTRY(reschedule_interrupt)
530 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
531
532ENTRY(invalidate_interrupt)
533 apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
534
535ENTRY(call_function_interrupt)
536 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
537#endif
538
539#ifdef CONFIG_X86_LOCAL_APIC
540ENTRY(apic_timer_interrupt)
541 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
542
543ENTRY(error_interrupt)
544 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
545
546ENTRY(spurious_interrupt)
547 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
548#endif
549
550/*
551 * Exception entry points.
552 */
553 .macro zeroentry sym
554 pushq $0 /* push error code/oldrax */
555 pushq %rax /* push real oldrax to the rdi slot */
556 leaq \sym(%rip),%rax
557 jmp error_entry
558 .endm
559
560 .macro errorentry sym
561 pushq %rax
562 leaq \sym(%rip),%rax
563 jmp error_entry
564 .endm
565
566 /* error code is on the stack already */
567 /* handle NMI like exceptions that can happen everywhere */
568 .macro paranoidentry sym
569 SAVE_ALL
570 cld
571 movl $1,%ebx
572 movl $MSR_GS_BASE,%ecx
573 rdmsr
574 testl %edx,%edx
575 js 1f
576 swapgs
577 xorl %ebx,%ebx
5781: movq %rsp,%rdi
579 movq ORIG_RAX(%rsp),%rsi
580 movq $-1,ORIG_RAX(%rsp)
581 call \sym
6fefb0d1 582 cli
1da177e4
LT		 583 .endm
584
585/*
586 * Exception entry point. This expects an error code/orig_rax on the stack
587 * and the exception handler in %rax.
588 */
589ENTRY(error_entry)
590 CFI_STARTPROC simple
591 CFI_DEF_CFA rsp,(SS-RDI)
592 CFI_REL_OFFSET rsp,(RSP-RDI)
593 CFI_REL_OFFSET rip,(RIP-RDI)
594 /* rdi slot contains rax, oldrax contains error code */
595 cld
596 subq $14*8,%rsp
597 CFI_ADJUST_CFA_OFFSET (14*8)
598 movq %rsi,13*8(%rsp)
599 CFI_REL_OFFSET rsi,RSI
600 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
601 movq %rdx,12*8(%rsp)
602 CFI_REL_OFFSET rdx,RDX
603 movq %rcx,11*8(%rsp)
604 CFI_REL_OFFSET rcx,RCX
605 movq %rsi,10*8(%rsp) /* store rax */
606 CFI_REL_OFFSET rax,RAX
607 movq %r8, 9*8(%rsp)
608 CFI_REL_OFFSET r8,R8
609 movq %r9, 8*8(%rsp)
610 CFI_REL_OFFSET r9,R9
611 movq %r10,7*8(%rsp)
612 CFI_REL_OFFSET r10,R10
613 movq %r11,6*8(%rsp)
614 CFI_REL_OFFSET r11,R11
615 movq %rbx,5*8(%rsp)
616 CFI_REL_OFFSET rbx,RBX
617 movq %rbp,4*8(%rsp)
618 CFI_REL_OFFSET rbp,RBP
619 movq %r12,3*8(%rsp)
620 CFI_REL_OFFSET r12,R12
621 movq %r13,2*8(%rsp)
622 CFI_REL_OFFSET r13,R13
623 movq %r14,1*8(%rsp)
624 CFI_REL_OFFSET r14,R14
625 movq %r15,(%rsp)
626 CFI_REL_OFFSET r15,R15
627 xorl %ebx,%ebx
628 testl $3,CS(%rsp)
629 je error_kernelspace
630error_swapgs:
631 swapgs
632error_sti:
633 movq %rdi,RDI(%rsp)
634 movq %rsp,%rdi
635 movq ORIG_RAX(%rsp),%rsi /* get error code */
636 movq $-1,ORIG_RAX(%rsp)
637 call *%rax
638 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
639error_exit:
640 movl %ebx,%eax
641 RESTORE_REST
642 cli
643 GET_THREAD_INFO(%rcx)
644 testl %eax,%eax
645 jne retint_kernel
646 movl threadinfo_flags(%rcx),%edx
647 movl $_TIF_WORK_MASK,%edi
648 andl %edi,%edx
649 jnz retint_careful
650 swapgs
651 RESTORE_ARGS 0,8,0
652 iretq
653 CFI_ENDPROC
654
655error_kernelspace:
656 incl %ebx
657 /* There are two places in the kernel that can potentially fault with
658 usergs. Handle them here. The exception handlers after
659 iret run with kernel gs again, so don't set the user space flag.
660 B stepping K8s sometimes report an truncated RIP for IRET
661 exceptions returning to compat mode. Check for these here too. */
662 leaq iret_label(%rip),%rbp
663 cmpq %rbp,RIP(%rsp)
664 je error_swapgs
665 movl %ebp,%ebp /* zero extend */
666 cmpq %rbp,RIP(%rsp)
667 je error_swapgs
668 cmpq $gs_change,RIP(%rsp)
669 je error_swapgs
670 jmp error_sti
671
672 /* Reload gs selector with exception handling */
673 /* edi: new selector */
674ENTRY(load_gs_index)
675 pushf
676 cli
677 swapgs
678gs_change:
679 movl %edi,%gs
6802: mfence /* workaround */
681 swapgs
682 popf
683 ret
684
685 .section __ex_table,"a"
686 .align 8
687 .quad gs_change,bad_gs
688 .previous
689 .section .fixup,"ax"
690 /* running with kernelgs */
691bad_gs:
692 swapgs /* switch back to user gs */
693 xorl %eax,%eax
694 movl %eax,%gs
695 jmp 2b
696 .previous
697
698/*
699 * Create a kernel thread.
700 *
701 * C extern interface:
702 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
703 *
704 * asm input arguments:
705 * rdi: fn, rsi: arg, rdx: flags
706 */
707ENTRY(kernel_thread)
708 CFI_STARTPROC
709 FAKE_STACK_FRAME $child_rip
710 SAVE_ALL
711
712 # rdi: flags, rsi: usp, rdx: will be &pt_regs
713 movq %rdx,%rdi
714 orq kernel_thread_flags(%rip),%rdi
715 movq $-1, %rsi
716 movq %rsp, %rdx
717
718 xorl %r8d,%r8d
719 xorl %r9d,%r9d
720
721 # clone now
722 call do_fork
723 movq %rax,RAX(%rsp)
724 xorl %edi,%edi
725
726 /*
727 * It isn't worth to check for reschedule here,
728 * so internally to the x86_64 port you can rely on kernel_thread()
729 * not to reschedule the child before returning, this avoids the need
730 * of hacks for example to fork off the per-CPU idle tasks.
731 * [Hopefully no generic code relies on the reschedule -AK]
732 */
733 RESTORE_ALL
734 UNFAKE_STACK_FRAME
735 ret
736 CFI_ENDPROC
737
738
739child_rip:
740 /*
741 * Here we are in the child and the registers are set as they were
742 * at kernel_thread() invocation in the parent.
743 */
744 movq %rdi, %rax
745 movq %rsi, %rdi
746 call *%rax
747 # exit
748 xorq %rdi, %rdi
749 call do_exit
750
751/*
752 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
753 *
754 * C extern interface:
755 * extern long execve(char *name, char **argv, char **envp)
756 *
757 * asm input arguments:
758 * rdi: name, rsi: argv, rdx: envp
759 *
760 * We want to fallback into:
761 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
762 *
763 * do_sys_execve asm fallback arguments:
764 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
765 */
766ENTRY(execve)
767 CFI_STARTPROC
768 FAKE_STACK_FRAME $0
769 SAVE_ALL
770 call sys_execve
771 movq %rax, RAX(%rsp)
772 RESTORE_REST
773 testq %rax,%rax
774 je int_ret_from_sys_call
775 RESTORE_ARGS
776 UNFAKE_STACK_FRAME
777 ret
778 CFI_ENDPROC
779
780ENTRY(page_fault)
781 errorentry do_page_fault
782
783ENTRY(coprocessor_error)
784 zeroentry do_coprocessor_error
785
786ENTRY(simd_coprocessor_error)
787 zeroentry do_simd_coprocessor_error
788
789ENTRY(device_not_available)
790 zeroentry math_state_restore
791
792 /* runs on exception stack */
793ENTRY(debug)
794 CFI_STARTPROC
795 pushq $0
796 CFI_ADJUST_CFA_OFFSET 8
797 paranoidentry do_debug
1da177e4
LT		 798 jmp paranoid_exit
799 CFI_ENDPROC
800
801 /* runs on exception stack */
802ENTRY(nmi)
803 CFI_STARTPROC
804 pushq $-1
805 CFI_ADJUST_CFA_OFFSET 8
806 paranoidentry do_nmi
6fefb0d1
AK		 807 /*
808 * "Paranoid" exit path from exception stack.
809 * Paranoid because this is used by NMIs and cannot take
810 * any kernel state for granted.
811 * We don't do kernel preemption checks here, because only
812 * NMI should be common and it does not enable IRQs and
813 * cannot get reschedule ticks.
814 */
1da177e4
LT		 815 /* ebx: no swapgs flag */
816paranoid_exit:
817 testl %ebx,%ebx /* swapgs needed? */
818 jnz paranoid_restore
6fefb0d1
AK		 819 testl $3,CS(%rsp)
820 jnz paranoid_userspace
1da177e4 821paranoid_swapgs:
1da177e4
LT		 822 swapgs
823paranoid_restore:
824 RESTORE_ALL 8
825 iretq
826paranoid_userspace:
1da177e4 827 GET_THREAD_INFO(%rcx)
6fefb0d1
AK		 828 movl threadinfo_flags(%rcx),%ebx
829 andl $_TIF_WORK_MASK,%ebx
11b854b2 830 jz paranoid_swapgs
6fefb0d1
AK		 831 movq %rsp,%rdi /* &pt_regs */
832 call sync_regs
833 movq %rax,%rsp /* switch stack for scheduling */
834 testl $_TIF_NEED_RESCHED,%ebx
835 jnz paranoid_schedule
836 movl %ebx,%edx /* arg3: thread flags */
1da177e4 837 sti
6fefb0d1
AK		 838 xorl %esi,%esi /* arg2: oldset */
839 movq %rsp,%rdi /* arg1: &pt_regs */
1da177e4 840 call do_notify_resume
6fefb0d1
AK		 841 cli
842 jmp paranoid_userspace
843paranoid_schedule:
11b854b2
AK		 844 sti
845 call schedule
6fefb0d1
AK		 846 cli
847 jmp paranoid_userspace
1da177e4 848 CFI_ENDPROC
6fefb0d1 849
1da177e4
LT		 850ENTRY(int3)
851 zeroentry do_int3
852
853ENTRY(overflow)
854 zeroentry do_overflow
855
856ENTRY(bounds)
857 zeroentry do_bounds
858
859ENTRY(invalid_op)
860 zeroentry do_invalid_op
861
862ENTRY(coprocessor_segment_overrun)
863 zeroentry do_coprocessor_segment_overrun
864
865ENTRY(reserved)
866 zeroentry do_reserved
867
868 /* runs on exception stack */
869ENTRY(double_fault)
870 CFI_STARTPROC
871 paranoidentry do_double_fault
1da177e4
LT		 872 jmp paranoid_exit
873 CFI_ENDPROC
874
875ENTRY(invalid_TSS)
876 errorentry do_invalid_TSS
877
878ENTRY(segment_not_present)
879 errorentry do_segment_not_present
880
881 /* runs on exception stack */
882ENTRY(stack_segment)
883 CFI_STARTPROC
884 paranoidentry do_stack_segment
1da177e4
LT		 885 jmp paranoid_exit
886 CFI_ENDPROC
887
888ENTRY(general_protection)
889 errorentry do_general_protection
890
891ENTRY(alignment_check)
892 errorentry do_alignment_check
893
894ENTRY(divide_error)
895 zeroentry do_divide_error
896
897ENTRY(spurious_interrupt_bug)
898 zeroentry do_spurious_interrupt_bug
899
900#ifdef CONFIG_X86_MCE
901 /* runs on exception stack */
902ENTRY(machine_check)
903 CFI_STARTPROC
904 pushq $0
905 CFI_ADJUST_CFA_OFFSET 8
906 paranoidentry do_machine_check
907 jmp paranoid_exit
908 CFI_ENDPROC
909#endif
910
911ENTRY(call_debug)
912 zeroentry do_call_debug
913
kernel source for telekom puls (alcatel/mediatek)