Commit | Line | Data |
---|---|---|
3610cce8 | 1 | /* |
a53c8fab | 2 | * Copyright IBM Corp. 2007, 2011 |
3610cce8 MS |
3 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
4 | */ | |
5 | ||
6 | #include <linux/sched.h> | |
7 | #include <linux/kernel.h> | |
8 | #include <linux/errno.h> | |
5a0e3ad6 | 9 | #include <linux/gfp.h> |
3610cce8 MS |
10 | #include <linux/mm.h> |
11 | #include <linux/swap.h> | |
12 | #include <linux/smp.h> | |
3610cce8 | 13 | #include <linux/spinlock.h> |
80217147 | 14 | #include <linux/rcupdate.h> |
e5992f2e | 15 | #include <linux/slab.h> |
b31288fa | 16 | #include <linux/swapops.h> |
0b46e0a3 | 17 | #include <linux/sysctl.h> |
3ac8e380 DD |
18 | #include <linux/ksm.h> |
19 | #include <linux/mman.h> | |
3610cce8 | 20 | |
3610cce8 MS |
21 | #include <asm/pgtable.h> |
22 | #include <asm/pgalloc.h> | |
23 | #include <asm/tlb.h> | |
24 | #include <asm/tlbflush.h> | |
6252d702 | 25 | #include <asm/mmu_context.h> |
2d42f947 | 26 | #include <asm/page-states.h> |
3610cce8 | 27 | |
118bd31b | 28 | static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, |
cd774b90 | 29 | pte_t *ptep, int nodat) |
118bd31b MS |
30 | { |
31 | unsigned long opt, asce; | |
32 | ||
33 | if (MACHINE_HAS_TLB_GUEST) { | |
34 | opt = 0; | |
35 | asce = READ_ONCE(mm->context.gmap_asce); | |
cd774b90 | 36 | if (asce == 0UL || nodat) |
118bd31b | 37 | opt |= IPTE_NODAT; |
28c807e5 MS |
38 | if (asce != -1UL) { |
39 | asce = asce ? : mm->context.asce; | |
40 | opt |= IPTE_GUEST_ASCE; | |
41 | } | |
42 | __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); | |
118bd31b | 43 | } else { |
28c807e5 | 44 | __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); |
118bd31b MS |
45 | } |
46 | } | |
47 | ||
48 | static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, | |
cd774b90 | 49 | pte_t *ptep, int nodat) |
118bd31b MS |
50 | { |
51 | unsigned long opt, asce; | |
52 | ||
53 | if (MACHINE_HAS_TLB_GUEST) { | |
54 | opt = 0; | |
55 | asce = READ_ONCE(mm->context.gmap_asce); | |
cd774b90 | 56 | if (asce == 0UL || nodat) |
118bd31b | 57 | opt |= IPTE_NODAT; |
28c807e5 MS |
58 | if (asce != -1UL) { |
59 | asce = asce ? : mm->context.asce; | |
60 | opt |= IPTE_GUEST_ASCE; | |
61 | } | |
62 | __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); | |
118bd31b | 63 | } else { |
28c807e5 | 64 | __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); |
118bd31b MS |
65 | } |
66 | } | |
67 | ||
ebde765c | 68 | static inline pte_t ptep_flush_direct(struct mm_struct *mm, |
cd774b90 MS |
69 | unsigned long addr, pte_t *ptep, |
70 | int nodat) | |
ebde765c | 71 | { |
ebde765c MS |
72 | pte_t old; |
73 | ||
74 | old = *ptep; | |
75 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
76 | return old; | |
64f31d58 MS |
77 | atomic_inc(&mm->context.flush_count); |
78 | if (MACHINE_HAS_TLB_LC && | |
ebde765c | 79 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
cd774b90 | 80 | ptep_ipte_local(mm, addr, ptep, nodat); |
ebde765c | 81 | else |
cd774b90 | 82 | ptep_ipte_global(mm, addr, ptep, nodat); |
64f31d58 | 83 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
84 | return old; |
85 | } | |
86 | ||
87 | static inline pte_t ptep_flush_lazy(struct mm_struct *mm, | |
cd774b90 MS |
88 | unsigned long addr, pte_t *ptep, |
89 | int nodat) | |
ebde765c | 90 | { |
ebde765c MS |
91 | pte_t old; |
92 | ||
93 | old = *ptep; | |
94 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
95 | return old; | |
64f31d58 MS |
96 | atomic_inc(&mm->context.flush_count); |
97 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
98 | cpumask_of(smp_processor_id()))) { | |
ebde765c MS |
99 | pte_val(*ptep) |= _PAGE_INVALID; |
100 | mm->context.flush_mm = 1; | |
101 | } else | |
cd774b90 | 102 | ptep_ipte_global(mm, addr, ptep, nodat); |
64f31d58 | 103 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
104 | return old; |
105 | } | |
106 | ||
1e133ab2 MS |
107 | static inline pgste_t pgste_get_lock(pte_t *ptep) |
108 | { | |
109 | unsigned long new = 0; | |
110 | #ifdef CONFIG_PGSTE | |
111 | unsigned long old; | |
112 | ||
1e133ab2 MS |
113 | asm( |
114 | " lg %0,%2\n" | |
115 | "0: lgr %1,%0\n" | |
116 | " nihh %0,0xff7f\n" /* clear PCL bit in old */ | |
117 | " oihh %1,0x0080\n" /* set PCL bit in new */ | |
118 | " csg %0,%1,%2\n" | |
119 | " jl 0b\n" | |
120 | : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) | |
121 | : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); | |
122 | #endif | |
123 | return __pgste(new); | |
124 | } | |
125 | ||
126 | static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) | |
127 | { | |
128 | #ifdef CONFIG_PGSTE | |
129 | asm( | |
130 | " nihh %1,0xff7f\n" /* clear PCL bit */ | |
131 | " stg %1,%0\n" | |
132 | : "=Q" (ptep[PTRS_PER_PTE]) | |
133 | : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) | |
134 | : "cc", "memory"); | |
1e133ab2 MS |
135 | #endif |
136 | } | |
137 | ||
138 | static inline pgste_t pgste_get(pte_t *ptep) | |
139 | { | |
140 | unsigned long pgste = 0; | |
141 | #ifdef CONFIG_PGSTE | |
142 | pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); | |
143 | #endif | |
144 | return __pgste(pgste); | |
145 | } | |
146 | ||
147 | static inline void pgste_set(pte_t *ptep, pgste_t pgste) | |
148 | { | |
149 | #ifdef CONFIG_PGSTE | |
150 | *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; | |
151 | #endif | |
152 | } | |
153 | ||
ebde765c MS |
154 | static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, |
155 | struct mm_struct *mm) | |
156 | { | |
157 | #ifdef CONFIG_PGSTE | |
158 | unsigned long address, bits, skey; | |
159 | ||
160 | if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID) | |
161 | return pgste; | |
162 | address = pte_val(pte) & PAGE_MASK; | |
163 | skey = (unsigned long) page_get_storage_key(address); | |
164 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
165 | /* Transfer page changed & referenced bit to guest bits in pgste */ | |
166 | pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ | |
167 | /* Copy page access key and fetch protection bit to pgste */ | |
168 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
169 | pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
170 | #endif | |
171 | return pgste; | |
172 | ||
173 | } | |
174 | ||
175 | static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, | |
176 | struct mm_struct *mm) | |
177 | { | |
178 | #ifdef CONFIG_PGSTE | |
179 | unsigned long address; | |
180 | unsigned long nkey; | |
181 | ||
182 | if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID) | |
183 | return; | |
184 | VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); | |
185 | address = pte_val(entry) & PAGE_MASK; | |
186 | /* | |
187 | * Set page access key and fetch protection bit from pgste. | |
188 | * The guest C/R information is still in the PGSTE, set real | |
189 | * key C/R to 0. | |
190 | */ | |
191 | nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; | |
192 | nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; | |
193 | page_set_storage_key(address, nkey, 0); | |
194 | #endif | |
195 | } | |
196 | ||
197 | static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) | |
198 | { | |
199 | #ifdef CONFIG_PGSTE | |
200 | if ((pte_val(entry) & _PAGE_PRESENT) && | |
201 | (pte_val(entry) & _PAGE_WRITE) && | |
202 | !(pte_val(entry) & _PAGE_INVALID)) { | |
203 | if (!MACHINE_HAS_ESOP) { | |
204 | /* | |
205 | * Without enhanced suppression-on-protection force | |
206 | * the dirty bit on for all writable ptes. | |
207 | */ | |
208 | pte_val(entry) |= _PAGE_DIRTY; | |
209 | pte_val(entry) &= ~_PAGE_PROTECT; | |
210 | } | |
211 | if (!(pte_val(entry) & _PAGE_PROTECT)) | |
212 | /* This pte allows write access, set user-dirty */ | |
213 | pgste_val(pgste) |= PGSTE_UC_BIT; | |
214 | } | |
215 | #endif | |
216 | *ptep = entry; | |
217 | return pgste; | |
218 | } | |
219 | ||
b2d73b2a MS |
220 | static inline pgste_t pgste_pte_notify(struct mm_struct *mm, |
221 | unsigned long addr, | |
222 | pte_t *ptep, pgste_t pgste) | |
ebde765c MS |
223 | { |
224 | #ifdef CONFIG_PGSTE | |
4be130a0 MS |
225 | unsigned long bits; |
226 | ||
227 | bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT); | |
228 | if (bits) { | |
229 | pgste_val(pgste) ^= bits; | |
230 | ptep_notify(mm, addr, ptep, bits); | |
ebde765c MS |
231 | } |
232 | #endif | |
233 | return pgste; | |
234 | } | |
235 | ||
ebde765c MS |
236 | static inline pgste_t ptep_xchg_start(struct mm_struct *mm, |
237 | unsigned long addr, pte_t *ptep) | |
238 | { | |
239 | pgste_t pgste = __pgste(0); | |
240 | ||
241 | if (mm_has_pgste(mm)) { | |
242 | pgste = pgste_get_lock(ptep); | |
b2d73b2a | 243 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
ebde765c MS |
244 | } |
245 | return pgste; | |
246 | } | |
247 | ||
0d6da872 | 248 | static inline pte_t ptep_xchg_commit(struct mm_struct *mm, |
ebde765c MS |
249 | unsigned long addr, pte_t *ptep, |
250 | pgste_t pgste, pte_t old, pte_t new) | |
251 | { | |
252 | if (mm_has_pgste(mm)) { | |
253 | if (pte_val(old) & _PAGE_INVALID) | |
254 | pgste_set_key(ptep, pgste, new, mm); | |
255 | if (pte_val(new) & _PAGE_INVALID) { | |
256 | pgste = pgste_update_all(old, pgste, mm); | |
257 | if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == | |
258 | _PGSTE_GPS_USAGE_UNUSED) | |
259 | pte_val(old) |= _PAGE_UNUSED; | |
260 | } | |
261 | pgste = pgste_set_pte(ptep, pgste, new); | |
262 | pgste_set_unlock(ptep, pgste); | |
263 | } else { | |
264 | *ptep = new; | |
265 | } | |
0d6da872 | 266 | return old; |
ebde765c MS |
267 | } |
268 | ||
269 | pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
270 | pte_t *ptep, pte_t new) | |
271 | { | |
272 | pgste_t pgste; | |
273 | pte_t old; | |
cd774b90 | 274 | int nodat; |
ebde765c | 275 | |
a9809407 | 276 | preempt_disable(); |
ebde765c | 277 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
278 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
279 | old = ptep_flush_direct(mm, addr, ptep, nodat); | |
0d6da872 | 280 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 281 | preempt_enable(); |
ebde765c MS |
282 | return old; |
283 | } | |
284 | EXPORT_SYMBOL(ptep_xchg_direct); | |
285 | ||
286 | pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
287 | pte_t *ptep, pte_t new) | |
288 | { | |
289 | pgste_t pgste; | |
290 | pte_t old; | |
cd774b90 | 291 | int nodat; |
ebde765c | 292 | |
a9809407 | 293 | preempt_disable(); |
ebde765c | 294 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
295 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
296 | old = ptep_flush_lazy(mm, addr, ptep, nodat); | |
0d6da872 | 297 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 298 | preempt_enable(); |
ebde765c MS |
299 | return old; |
300 | } | |
301 | EXPORT_SYMBOL(ptep_xchg_lazy); | |
302 | ||
303 | pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, | |
304 | pte_t *ptep) | |
305 | { | |
306 | pgste_t pgste; | |
307 | pte_t old; | |
cd774b90 | 308 | int nodat; |
ebde765c | 309 | |
a9809407 | 310 | preempt_disable(); |
ebde765c | 311 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
312 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
313 | old = ptep_flush_lazy(mm, addr, ptep, nodat); | |
ebde765c MS |
314 | if (mm_has_pgste(mm)) { |
315 | pgste = pgste_update_all(old, pgste, mm); | |
316 | pgste_set(ptep, pgste); | |
317 | } | |
318 | return old; | |
319 | } | |
320 | EXPORT_SYMBOL(ptep_modify_prot_start); | |
321 | ||
322 | void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, | |
323 | pte_t *ptep, pte_t pte) | |
324 | { | |
325 | pgste_t pgste; | |
326 | ||
57d7f939 MS |
327 | if (!MACHINE_HAS_NX) |
328 | pte_val(pte) &= ~_PAGE_NOEXEC; | |
ebde765c MS |
329 | if (mm_has_pgste(mm)) { |
330 | pgste = pgste_get(ptep); | |
331 | pgste_set_key(ptep, pgste, pte, mm); | |
332 | pgste = pgste_set_pte(ptep, pgste, pte); | |
333 | pgste_set_unlock(ptep, pgste); | |
334 | } else { | |
335 | *ptep = pte; | |
336 | } | |
a9809407 | 337 | preempt_enable(); |
ebde765c MS |
338 | } |
339 | EXPORT_SYMBOL(ptep_modify_prot_commit); | |
340 | ||
118bd31b MS |
341 | static inline void pmdp_idte_local(struct mm_struct *mm, |
342 | unsigned long addr, pmd_t *pmdp) | |
343 | { | |
344 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
345 | __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, |
346 | mm->context.asce, IDTE_LOCAL); | |
118bd31b | 347 | else |
28c807e5 | 348 | __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); |
118bd31b MS |
349 | } |
350 | ||
351 | static inline void pmdp_idte_global(struct mm_struct *mm, | |
352 | unsigned long addr, pmd_t *pmdp) | |
353 | { | |
354 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
355 | __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, |
356 | mm->context.asce, IDTE_GLOBAL); | |
118bd31b | 357 | else if (MACHINE_HAS_IDTE) |
28c807e5 | 358 | __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); |
118bd31b MS |
359 | else |
360 | __pmdp_csp(pmdp); | |
361 | } | |
362 | ||
227be799 MS |
363 | static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, |
364 | unsigned long addr, pmd_t *pmdp) | |
365 | { | |
227be799 MS |
366 | pmd_t old; |
367 | ||
368 | old = *pmdp; | |
369 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
370 | return old; | |
64f31d58 MS |
371 | atomic_inc(&mm->context.flush_count); |
372 | if (MACHINE_HAS_TLB_LC && | |
227be799 | 373 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
118bd31b | 374 | pmdp_idte_local(mm, addr, pmdp); |
227be799 | 375 | else |
118bd31b | 376 | pmdp_idte_global(mm, addr, pmdp); |
64f31d58 | 377 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
378 | return old; |
379 | } | |
380 | ||
381 | static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, | |
382 | unsigned long addr, pmd_t *pmdp) | |
383 | { | |
227be799 MS |
384 | pmd_t old; |
385 | ||
386 | old = *pmdp; | |
387 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
388 | return old; | |
64f31d58 MS |
389 | atomic_inc(&mm->context.flush_count); |
390 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
391 | cpumask_of(smp_processor_id()))) { | |
227be799 MS |
392 | pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; |
393 | mm->context.flush_mm = 1; | |
118bd31b MS |
394 | } else { |
395 | pmdp_idte_global(mm, addr, pmdp); | |
396 | } | |
64f31d58 | 397 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
398 | return old; |
399 | } | |
400 | ||
401 | pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
402 | pmd_t *pmdp, pmd_t new) | |
403 | { | |
404 | pmd_t old; | |
405 | ||
a9809407 | 406 | preempt_disable(); |
227be799 MS |
407 | old = pmdp_flush_direct(mm, addr, pmdp); |
408 | *pmdp = new; | |
a9809407 | 409 | preempt_enable(); |
227be799 MS |
410 | return old; |
411 | } | |
412 | EXPORT_SYMBOL(pmdp_xchg_direct); | |
413 | ||
414 | pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
415 | pmd_t *pmdp, pmd_t new) | |
416 | { | |
417 | pmd_t old; | |
418 | ||
a9809407 | 419 | preempt_disable(); |
227be799 MS |
420 | old = pmdp_flush_lazy(mm, addr, pmdp); |
421 | *pmdp = new; | |
a9809407 | 422 | preempt_enable(); |
227be799 MS |
423 | return old; |
424 | } | |
425 | EXPORT_SYMBOL(pmdp_xchg_lazy); | |
426 | ||
118bd31b MS |
427 | static inline void pudp_idte_local(struct mm_struct *mm, |
428 | unsigned long addr, pud_t *pudp) | |
d08de8e2 | 429 | { |
118bd31b | 430 | if (MACHINE_HAS_TLB_GUEST) |
28c807e5 MS |
431 | __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, |
432 | mm->context.asce, IDTE_LOCAL); | |
118bd31b | 433 | else |
28c807e5 | 434 | __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); |
118bd31b | 435 | } |
d08de8e2 | 436 | |
118bd31b MS |
437 | static inline void pudp_idte_global(struct mm_struct *mm, |
438 | unsigned long addr, pud_t *pudp) | |
439 | { | |
440 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
441 | __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, |
442 | mm->context.asce, IDTE_GLOBAL); | |
118bd31b | 443 | else if (MACHINE_HAS_IDTE) |
28c807e5 | 444 | __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); |
118bd31b | 445 | else |
d08de8e2 GS |
446 | /* |
447 | * Invalid bit position is the same for pmd and pud, so we can | |
448 | * re-use _pmd_csp() here | |
449 | */ | |
450 | __pmdp_csp((pmd_t *) pudp); | |
118bd31b MS |
451 | } |
452 | ||
453 | static inline pud_t pudp_flush_direct(struct mm_struct *mm, | |
454 | unsigned long addr, pud_t *pudp) | |
455 | { | |
456 | pud_t old; | |
457 | ||
458 | old = *pudp; | |
459 | if (pud_val(old) & _REGION_ENTRY_INVALID) | |
d08de8e2 | 460 | return old; |
d08de8e2 GS |
461 | atomic_inc(&mm->context.flush_count); |
462 | if (MACHINE_HAS_TLB_LC && | |
463 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) | |
118bd31b | 464 | pudp_idte_local(mm, addr, pudp); |
d08de8e2 | 465 | else |
118bd31b | 466 | pudp_idte_global(mm, addr, pudp); |
d08de8e2 GS |
467 | atomic_dec(&mm->context.flush_count); |
468 | return old; | |
469 | } | |
470 | ||
471 | pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
472 | pud_t *pudp, pud_t new) | |
473 | { | |
474 | pud_t old; | |
475 | ||
476 | preempt_disable(); | |
477 | old = pudp_flush_direct(mm, addr, pudp); | |
478 | *pudp = new; | |
479 | preempt_enable(); | |
480 | return old; | |
481 | } | |
482 | EXPORT_SYMBOL(pudp_xchg_direct); | |
483 | ||
75077afb | 484 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
6b0b50b0 AK |
485 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
486 | pgtable_t pgtable) | |
9501d09f GS |
487 | { |
488 | struct list_head *lh = (struct list_head *) pgtable; | |
489 | ||
ec66ad66 | 490 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
491 | |
492 | /* FIFO */ | |
c389a250 | 493 | if (!pmd_huge_pte(mm, pmdp)) |
9501d09f GS |
494 | INIT_LIST_HEAD(lh); |
495 | else | |
c389a250 KS |
496 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
497 | pmd_huge_pte(mm, pmdp) = pgtable; | |
9501d09f GS |
498 | } |
499 | ||
6b0b50b0 | 500 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
9501d09f GS |
501 | { |
502 | struct list_head *lh; | |
503 | pgtable_t pgtable; | |
504 | pte_t *ptep; | |
505 | ||
ec66ad66 | 506 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
507 | |
508 | /* FIFO */ | |
c389a250 | 509 | pgtable = pmd_huge_pte(mm, pmdp); |
9501d09f GS |
510 | lh = (struct list_head *) pgtable; |
511 | if (list_empty(lh)) | |
c389a250 | 512 | pmd_huge_pte(mm, pmdp) = NULL; |
9501d09f | 513 | else { |
c389a250 | 514 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
9501d09f GS |
515 | list_del(lh); |
516 | } | |
517 | ptep = (pte_t *) pgtable; | |
e5098611 | 518 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f | 519 | ptep++; |
e5098611 | 520 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f GS |
521 | return pgtable; |
522 | } | |
75077afb | 523 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
1e133ab2 MS |
524 | |
525 | #ifdef CONFIG_PGSTE | |
526 | void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, | |
527 | pte_t *ptep, pte_t entry) | |
528 | { | |
529 | pgste_t pgste; | |
530 | ||
531 | /* the mm_has_pgste() check is done in set_pte_at() */ | |
a9809407 | 532 | preempt_disable(); |
1e133ab2 MS |
533 | pgste = pgste_get_lock(ptep); |
534 | pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; | |
535 | pgste_set_key(ptep, pgste, entry, mm); | |
536 | pgste = pgste_set_pte(ptep, pgste, entry); | |
537 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 538 | preempt_enable(); |
1e133ab2 MS |
539 | } |
540 | ||
541 | void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
542 | { | |
543 | pgste_t pgste; | |
544 | ||
a9809407 | 545 | preempt_disable(); |
1e133ab2 MS |
546 | pgste = pgste_get_lock(ptep); |
547 | pgste_val(pgste) |= PGSTE_IN_BIT; | |
548 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 549 | preempt_enable(); |
1e133ab2 MS |
550 | } |
551 | ||
b2d73b2a MS |
552 | /** |
553 | * ptep_force_prot - change access rights of a locked pte | |
554 | * @mm: pointer to the process mm_struct | |
555 | * @addr: virtual address in the guest address space | |
556 | * @ptep: pointer to the page table entry | |
557 | * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE | |
4be130a0 | 558 | * @bit: pgste bit to set (e.g. for notification) |
b2d73b2a MS |
559 | * |
560 | * Returns 0 if the access rights were changed and -EAGAIN if the current | |
561 | * and requested access rights are incompatible. | |
562 | */ | |
563 | int ptep_force_prot(struct mm_struct *mm, unsigned long addr, | |
4be130a0 | 564 | pte_t *ptep, int prot, unsigned long bit) |
b2d73b2a MS |
565 | { |
566 | pte_t entry; | |
567 | pgste_t pgste; | |
cd774b90 | 568 | int pte_i, pte_p, nodat; |
b2d73b2a MS |
569 | |
570 | pgste = pgste_get_lock(ptep); | |
571 | entry = *ptep; | |
572 | /* Check pte entry after all locks have been acquired */ | |
573 | pte_i = pte_val(entry) & _PAGE_INVALID; | |
574 | pte_p = pte_val(entry) & _PAGE_PROTECT; | |
575 | if ((pte_i && (prot != PROT_NONE)) || | |
576 | (pte_p && (prot & PROT_WRITE))) { | |
577 | pgste_set_unlock(ptep, pgste); | |
578 | return -EAGAIN; | |
579 | } | |
4be130a0 | 580 | /* Change access rights and set pgste bit */ |
cd774b90 | 581 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
b2d73b2a | 582 | if (prot == PROT_NONE && !pte_i) { |
cd774b90 | 583 | ptep_flush_direct(mm, addr, ptep, nodat); |
b2d73b2a MS |
584 | pgste = pgste_update_all(entry, pgste, mm); |
585 | pte_val(entry) |= _PAGE_INVALID; | |
586 | } | |
587 | if (prot == PROT_READ && !pte_p) { | |
cd774b90 | 588 | ptep_flush_direct(mm, addr, ptep, nodat); |
b2d73b2a MS |
589 | pte_val(entry) &= ~_PAGE_INVALID; |
590 | pte_val(entry) |= _PAGE_PROTECT; | |
591 | } | |
4be130a0 | 592 | pgste_val(pgste) |= bit; |
b2d73b2a MS |
593 | pgste = pgste_set_pte(ptep, pgste, entry); |
594 | pgste_set_unlock(ptep, pgste); | |
595 | return 0; | |
596 | } | |
597 | ||
4be130a0 | 598 | int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, |
a9d23e71 | 599 | pte_t *sptep, pte_t *tptep, pte_t pte) |
4be130a0 MS |
600 | { |
601 | pgste_t spgste, tpgste; | |
602 | pte_t spte, tpte; | |
603 | int rc = -EAGAIN; | |
604 | ||
a9d23e71 DH |
605 | if (!(pte_val(*tptep) & _PAGE_INVALID)) |
606 | return 0; /* already shadowed */ | |
4be130a0 MS |
607 | spgste = pgste_get_lock(sptep); |
608 | spte = *sptep; | |
609 | if (!(pte_val(spte) & _PAGE_INVALID) && | |
a9d23e71 DH |
610 | !((pte_val(spte) & _PAGE_PROTECT) && |
611 | !(pte_val(pte) & _PAGE_PROTECT))) { | |
4be130a0 MS |
612 | pgste_val(spgste) |= PGSTE_VSIE_BIT; |
613 | tpgste = pgste_get_lock(tptep); | |
614 | pte_val(tpte) = (pte_val(spte) & PAGE_MASK) | | |
a9d23e71 | 615 | (pte_val(pte) & _PAGE_PROTECT); |
4be130a0 MS |
616 | /* don't touch the storage key - it belongs to parent pgste */ |
617 | tpgste = pgste_set_pte(tptep, tpgste, tpte); | |
618 | pgste_set_unlock(tptep, tpgste); | |
a9d23e71 | 619 | rc = 1; |
4be130a0 MS |
620 | } |
621 | pgste_set_unlock(sptep, spgste); | |
622 | return rc; | |
623 | } | |
624 | ||
625 | void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) | |
626 | { | |
627 | pgste_t pgste; | |
cd774b90 | 628 | int nodat; |
4be130a0 MS |
629 | |
630 | pgste = pgste_get_lock(ptep); | |
631 | /* notifier is called by the caller */ | |
cd774b90 MS |
632 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
633 | ptep_flush_direct(mm, saddr, ptep, nodat); | |
4be130a0 MS |
634 | /* don't touch the storage key - it belongs to parent pgste */ |
635 | pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); | |
636 | pgste_set_unlock(ptep, pgste); | |
637 | } | |
638 | ||
1e133ab2 MS |
639 | static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) |
640 | { | |
641 | if (!non_swap_entry(entry)) | |
642 | dec_mm_counter(mm, MM_SWAPENTS); | |
643 | else if (is_migration_entry(entry)) { | |
644 | struct page *page = migration_entry_to_page(entry); | |
645 | ||
646 | dec_mm_counter(mm, mm_counter(page)); | |
647 | } | |
648 | free_swap_and_cache(entry); | |
649 | } | |
650 | ||
651 | void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, | |
652 | pte_t *ptep, int reset) | |
653 | { | |
654 | unsigned long pgstev; | |
655 | pgste_t pgste; | |
656 | pte_t pte; | |
657 | ||
658 | /* Zap unused and logically-zero pages */ | |
a9809407 | 659 | preempt_disable(); |
1e133ab2 MS |
660 | pgste = pgste_get_lock(ptep); |
661 | pgstev = pgste_val(pgste); | |
662 | pte = *ptep; | |
1c343f7b | 663 | if (!reset && pte_swap(pte) && |
1e133ab2 MS |
664 | ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || |
665 | (pgstev & _PGSTE_GPS_ZERO))) { | |
666 | ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); | |
667 | pte_clear(mm, addr, ptep); | |
668 | } | |
669 | if (reset) | |
670 | pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; | |
671 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 672 | preempt_enable(); |
1e133ab2 MS |
673 | } |
674 | ||
675 | void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
676 | { | |
677 | unsigned long ptev; | |
678 | pgste_t pgste; | |
679 | ||
97ca7bfc | 680 | /* Clear storage key ACC and F, but set R/C */ |
a9809407 | 681 | preempt_disable(); |
1e133ab2 | 682 | pgste = pgste_get_lock(ptep); |
97ca7bfc CB |
683 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); |
684 | pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT; | |
1e133ab2 MS |
685 | ptev = pte_val(*ptep); |
686 | if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) | |
687 | page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); | |
688 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 689 | preempt_enable(); |
1e133ab2 MS |
690 | } |
691 | ||
692 | /* | |
693 | * Test and reset if a guest page is dirty | |
694 | */ | |
695 | bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) | |
696 | { | |
697 | spinlock_t *ptl; | |
2e4d8800 | 698 | pgd_t *pgd; |
1aea9b3f | 699 | p4d_t *p4d; |
2e4d8800 JF |
700 | pud_t *pud; |
701 | pmd_t *pmd; | |
1e133ab2 MS |
702 | pgste_t pgste; |
703 | pte_t *ptep; | |
704 | pte_t pte; | |
705 | bool dirty; | |
cd774b90 | 706 | int nodat; |
1e133ab2 | 707 | |
2e4d8800 | 708 | pgd = pgd_offset(mm, addr); |
1aea9b3f MS |
709 | p4d = p4d_alloc(mm, pgd, addr); |
710 | if (!p4d) | |
711 | return false; | |
712 | pud = pud_alloc(mm, p4d, addr); | |
2e4d8800 JF |
713 | if (!pud) |
714 | return false; | |
715 | pmd = pmd_alloc(mm, pud, addr); | |
716 | if (!pmd) | |
717 | return false; | |
718 | /* We can't run guests backed by huge pages, but userspace can | |
719 | * still set them up and then try to migrate them without any | |
720 | * migration support. | |
721 | */ | |
722 | if (pmd_large(*pmd)) | |
723 | return true; | |
724 | ||
725 | ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl); | |
1e133ab2 MS |
726 | if (unlikely(!ptep)) |
727 | return false; | |
728 | ||
729 | pgste = pgste_get_lock(ptep); | |
730 | dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); | |
731 | pgste_val(pgste) &= ~PGSTE_UC_BIT; | |
732 | pte = *ptep; | |
733 | if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { | |
b2d73b2a | 734 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
cd774b90 MS |
735 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
736 | ptep_ipte_global(mm, addr, ptep, nodat); | |
1e133ab2 MS |
737 | if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) |
738 | pte_val(pte) |= _PAGE_PROTECT; | |
739 | else | |
740 | pte_val(pte) |= _PAGE_INVALID; | |
741 | *ptep = pte; | |
742 | } | |
743 | pgste_set_unlock(ptep, pgste); | |
744 | ||
745 | spin_unlock(ptl); | |
746 | return dirty; | |
747 | } | |
748 | EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty); | |
749 | ||
750 | int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
751 | unsigned char key, bool nq) | |
752 | { | |
753 | unsigned long keyul; | |
754 | spinlock_t *ptl; | |
755 | pgste_t old, new; | |
756 | pte_t *ptep; | |
757 | ||
1e133ab2 | 758 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 759 | if (unlikely(!ptep)) |
1e133ab2 | 760 | return -EFAULT; |
1e133ab2 MS |
761 | |
762 | new = old = pgste_get_lock(ptep); | |
763 | pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT | | |
764 | PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
765 | keyul = (unsigned long) key; | |
766 | pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; | |
767 | pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
768 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
769 | unsigned long address, bits, skey; | |
770 | ||
771 | address = pte_val(*ptep) & PAGE_MASK; | |
772 | skey = (unsigned long) page_get_storage_key(address); | |
773 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
774 | skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); | |
775 | /* Set storage key ACC and FP */ | |
776 | page_set_storage_key(address, skey, !nq); | |
777 | /* Merge host changed & referenced into pgste */ | |
778 | pgste_val(new) |= bits << 52; | |
779 | } | |
780 | /* changing the guest storage key is considered a change of the page */ | |
781 | if ((pgste_val(new) ^ pgste_val(old)) & | |
782 | (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) | |
783 | pgste_val(new) |= PGSTE_UC_BIT; | |
784 | ||
785 | pgste_set_unlock(ptep, new); | |
786 | pte_unmap_unlock(ptep, ptl); | |
1e133ab2 MS |
787 | return 0; |
788 | } | |
789 | EXPORT_SYMBOL(set_guest_storage_key); | |
790 | ||
1824c723 DH |
791 | /** |
792 | * Conditionally set a guest storage key (handling csske). | |
793 | * oldkey will be updated when either mr or mc is set and a pointer is given. | |
794 | * | |
795 | * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest | |
796 | * storage key was updated and -EFAULT on access errors. | |
797 | */ | |
798 | int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
799 | unsigned char key, unsigned char *oldkey, | |
800 | bool nq, bool mr, bool mc) | |
801 | { | |
802 | unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT; | |
803 | int rc; | |
804 | ||
805 | /* we can drop the pgste lock between getting and setting the key */ | |
806 | if (mr | mc) { | |
807 | rc = get_guest_storage_key(current->mm, addr, &tmp); | |
808 | if (rc) | |
809 | return rc; | |
810 | if (oldkey) | |
811 | *oldkey = tmp; | |
812 | if (!mr) | |
813 | mask |= _PAGE_REFERENCED; | |
814 | if (!mc) | |
815 | mask |= _PAGE_CHANGED; | |
816 | if (!((tmp ^ key) & mask)) | |
817 | return 0; | |
818 | } | |
819 | rc = set_guest_storage_key(current->mm, addr, key, nq); | |
820 | return rc < 0 ? rc : 1; | |
821 | } | |
822 | EXPORT_SYMBOL(cond_set_guest_storage_key); | |
823 | ||
a7e19ab5 DH |
824 | /** |
825 | * Reset a guest reference bit (rrbe), returning the reference and changed bit. | |
826 | * | |
827 | * Returns < 0 in case of error, otherwise the cc to be reported to the guest. | |
828 | */ | |
829 | int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) | |
1e133ab2 | 830 | { |
1e133ab2 | 831 | spinlock_t *ptl; |
a7e19ab5 | 832 | pgste_t old, new; |
1e133ab2 | 833 | pte_t *ptep; |
a7e19ab5 | 834 | int cc = 0; |
1e133ab2 | 835 | |
1e133ab2 | 836 | ptep = get_locked_pte(mm, addr, &ptl); |
a7e19ab5 | 837 | if (unlikely(!ptep)) |
1e133ab2 | 838 | return -EFAULT; |
1e133ab2 | 839 | |
a7e19ab5 DH |
840 | new = old = pgste_get_lock(ptep); |
841 | /* Reset guest reference bit only */ | |
842 | pgste_val(new) &= ~PGSTE_GR_BIT; | |
1e133ab2 | 843 | |
a7e19ab5 DH |
844 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { |
845 | cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); | |
846 | /* Merge real referenced bit into host-set */ | |
847 | pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; | |
1e133ab2 | 848 | } |
a7e19ab5 DH |
849 | /* Reflect guest's logical view, not physical */ |
850 | cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49; | |
851 | /* Changing the guest storage key is considered a change of the page */ | |
852 | if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT) | |
853 | pgste_val(new) |= PGSTE_UC_BIT; | |
854 | ||
855 | pgste_set_unlock(ptep, new); | |
856 | pte_unmap_unlock(ptep, ptl); | |
4bead2a4 | 857 | return cc; |
a7e19ab5 DH |
858 | } |
859 | EXPORT_SYMBOL(reset_guest_reference_bit); | |
860 | ||
154c8c19 DH |
861 | int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, |
862 | unsigned char *key) | |
1e133ab2 | 863 | { |
1e133ab2 MS |
864 | spinlock_t *ptl; |
865 | pgste_t pgste; | |
866 | pte_t *ptep; | |
1e133ab2 | 867 | |
1e133ab2 | 868 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 869 | if (unlikely(!ptep)) |
1e133ab2 | 870 | return -EFAULT; |
1e133ab2 | 871 | |
d3ed1cee | 872 | pgste = pgste_get_lock(ptep); |
154c8c19 | 873 | *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; |
8d6037a7 | 874 | if (!(pte_val(*ptep) & _PAGE_INVALID)) |
154c8c19 | 875 | *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); |
8d6037a7 | 876 | /* Reflect guest's logical view, not physical */ |
154c8c19 | 877 | *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; |
1e133ab2 MS |
878 | pgste_set_unlock(ptep, pgste); |
879 | pte_unmap_unlock(ptep, ptl); | |
154c8c19 | 880 | return 0; |
1e133ab2 MS |
881 | } |
882 | EXPORT_SYMBOL(get_guest_storage_key); | |
2d42f947 CI |
883 | |
884 | /** | |
885 | * pgste_perform_essa - perform ESSA actions on the PGSTE. | |
886 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
887 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
888 | * @orc: the specific action to perform, see the ESSA_SET_* macros. | |
889 | * @oldpte: the PTE will be saved there if the pointer is not NULL. | |
890 | * @oldpgste: the old PGSTE will be saved there if the pointer is not NULL. | |
891 | * | |
892 | * Return: 1 if the page is to be added to the CBRL, otherwise 0, | |
893 | * or < 0 in case of error. -EINVAL is returned for invalid values | |
894 | * of orc, -EFAULT for invalid addresses. | |
895 | */ | |
896 | int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, | |
897 | unsigned long *oldpte, unsigned long *oldpgste) | |
898 | { | |
899 | unsigned long pgstev; | |
900 | spinlock_t *ptl; | |
901 | pgste_t pgste; | |
902 | pte_t *ptep; | |
903 | int res = 0; | |
904 | ||
905 | WARN_ON_ONCE(orc > ESSA_MAX); | |
906 | if (unlikely(orc > ESSA_MAX)) | |
907 | return -EINVAL; | |
908 | ptep = get_locked_pte(mm, hva, &ptl); | |
909 | if (unlikely(!ptep)) | |
910 | return -EFAULT; | |
911 | pgste = pgste_get_lock(ptep); | |
912 | pgstev = pgste_val(pgste); | |
913 | if (oldpte) | |
914 | *oldpte = pte_val(*ptep); | |
915 | if (oldpgste) | |
916 | *oldpgste = pgstev; | |
917 | ||
918 | switch (orc) { | |
919 | case ESSA_GET_STATE: | |
920 | break; | |
921 | case ESSA_SET_STABLE: | |
1bab1c02 | 922 | pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT); |
2d42f947 CI |
923 | pgstev |= _PGSTE_GPS_USAGE_STABLE; |
924 | break; | |
925 | case ESSA_SET_UNUSED: | |
926 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
927 | pgstev |= _PGSTE_GPS_USAGE_UNUSED; | |
928 | if (pte_val(*ptep) & _PAGE_INVALID) | |
929 | res = 1; | |
930 | break; | |
931 | case ESSA_SET_VOLATILE: | |
932 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
933 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
934 | if (pte_val(*ptep) & _PAGE_INVALID) | |
935 | res = 1; | |
936 | break; | |
937 | case ESSA_SET_POT_VOLATILE: | |
938 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
939 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
940 | pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE; | |
941 | break; | |
942 | } | |
943 | if (pgstev & _PGSTE_GPS_ZERO) { | |
944 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
945 | break; | |
946 | } | |
947 | if (!(pgstev & PGSTE_GC_BIT)) { | |
948 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
949 | res = 1; | |
950 | break; | |
951 | } | |
952 | break; | |
953 | case ESSA_SET_STABLE_RESIDENT: | |
954 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
955 | pgstev |= _PGSTE_GPS_USAGE_STABLE; | |
956 | /* | |
957 | * Since the resident state can go away any time after this | |
958 | * call, we will not make this page resident. We can revisit | |
959 | * this decision if a guest will ever start using this. | |
960 | */ | |
961 | break; | |
962 | case ESSA_SET_STABLE_IF_RESIDENT: | |
963 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
964 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
965 | pgstev |= _PGSTE_GPS_USAGE_STABLE; | |
966 | } | |
967 | break; | |
1bab1c02 CI |
968 | case ESSA_SET_STABLE_NODAT: |
969 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
970 | pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT; | |
971 | break; | |
2d42f947 CI |
972 | default: |
973 | /* we should never get here! */ | |
974 | break; | |
975 | } | |
976 | /* If we are discarding a page, set it to logical zero */ | |
977 | if (res) | |
978 | pgstev |= _PGSTE_GPS_ZERO; | |
979 | ||
980 | pgste_val(pgste) = pgstev; | |
981 | pgste_set_unlock(ptep, pgste); | |
982 | pte_unmap_unlock(ptep, ptl); | |
983 | return res; | |
984 | } | |
985 | EXPORT_SYMBOL(pgste_perform_essa); | |
986 | ||
987 | /** | |
988 | * set_pgste_bits - set specific PGSTE bits. | |
989 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
990 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
991 | * @bits: a bitmask representing the bits that will be touched | |
992 | * @value: the values of the bits to be written. Only the bits in the mask | |
993 | * will be written. | |
994 | * | |
995 | * Return: 0 on success, < 0 in case of error. | |
996 | */ | |
997 | int set_pgste_bits(struct mm_struct *mm, unsigned long hva, | |
998 | unsigned long bits, unsigned long value) | |
999 | { | |
1000 | spinlock_t *ptl; | |
1001 | pgste_t new; | |
1002 | pte_t *ptep; | |
1003 | ||
1004 | ptep = get_locked_pte(mm, hva, &ptl); | |
1005 | if (unlikely(!ptep)) | |
1006 | return -EFAULT; | |
1007 | new = pgste_get_lock(ptep); | |
1008 | ||
1009 | pgste_val(new) &= ~bits; | |
1010 | pgste_val(new) |= value & bits; | |
1011 | ||
1012 | pgste_set_unlock(ptep, new); | |
1013 | pte_unmap_unlock(ptep, ptl); | |
1014 | return 0; | |
1015 | } | |
1016 | EXPORT_SYMBOL(set_pgste_bits); | |
1017 | ||
1018 | /** | |
1019 | * get_pgste - get the current PGSTE for the given address. | |
1020 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
1021 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
1022 | * @pgstep: will be written with the current PGSTE for the given address. | |
1023 | * | |
1024 | * Return: 0 on success, < 0 in case of error. | |
1025 | */ | |
1026 | int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) | |
1027 | { | |
1028 | spinlock_t *ptl; | |
1029 | pte_t *ptep; | |
1030 | ||
1031 | ptep = get_locked_pte(mm, hva, &ptl); | |
1032 | if (unlikely(!ptep)) | |
1033 | return -EFAULT; | |
1034 | *pgstep = pgste_val(pgste_get(ptep)); | |
1035 | pte_unmap_unlock(ptep, ptl); | |
1036 | return 0; | |
1037 | } | |
1038 | EXPORT_SYMBOL(get_pgste); | |
1e133ab2 | 1039 | #endif |