Commit | Line | Data |
---|---|---|
453dc659 DT |
1 | /* |
2 | * VMware Balloon driver. | |
3 | * | |
48e3d668 | 4 | * Copyright (C) 2000-2014, VMware, Inc. All Rights Reserved. |
453dc659 DT |
5 | * |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License as published by the | |
8 | * Free Software Foundation; version 2 of the License and no later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
13 | * NON INFRINGEMENT. See the GNU General Public License for more | |
14 | * details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program; if not, write to the Free Software | |
18 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
19 | * | |
73b35d07 DT |
20 | * Maintained by: Xavier Deguillard <xdeguillard@vmware.com> |
21 | * Philip Moltmann <moltmann@vmware.com> | |
453dc659 DT |
22 | */ |
23 | ||
24 | /* | |
25 | * This is VMware physical memory management driver for Linux. The driver | |
26 | * acts like a "balloon" that can be inflated to reclaim physical pages by | |
27 | * reserving them in the guest and invalidating them in the monitor, | |
28 | * freeing up the underlying machine pages so they can be allocated to | |
29 | * other guests. The balloon can also be deflated to allow the guest to | |
30 | * use more physical memory. Higher level policies can control the sizes | |
31 | * of balloons in VMs in order to manage physical memory resources. | |
32 | */ | |
33 | ||
34 | //#define DEBUG | |
35 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
36 | ||
37 | #include <linux/types.h> | |
38 | #include <linux/kernel.h> | |
39 | #include <linux/mm.h> | |
f220a80f | 40 | #include <linux/vmalloc.h> |
453dc659 DT |
41 | #include <linux/sched.h> |
42 | #include <linux/module.h> | |
43 | #include <linux/workqueue.h> | |
44 | #include <linux/debugfs.h> | |
45 | #include <linux/seq_file.h> | |
48e3d668 PM |
46 | #include <linux/vmw_vmci_defs.h> |
47 | #include <linux/vmw_vmci_api.h> | |
956fa507 | 48 | #include <linux/io.h> |
a10a5698 | 49 | #include <asm/hypervisor.h> |
453dc659 DT |
50 | |
51 | MODULE_AUTHOR("VMware, Inc."); | |
52 | MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver"); | |
48e3d668 | 53 | MODULE_VERSION("1.5.0.0-k"); |
453dc659 DT |
54 | MODULE_ALIAS("dmi:*:svnVMware*:*"); |
55 | MODULE_ALIAS("vmware_vmmemctl"); | |
56 | MODULE_LICENSE("GPL"); | |
57 | ||
58 | /* | |
59 | * Various constants controlling rate of inflaint/deflating balloon, | |
60 | * measured in pages. | |
61 | */ | |
62 | ||
453dc659 DT |
63 | /* |
64 | * Rates of memory allocaton when guest experiences memory pressure | |
65 | * (driver performs sleeping allocations). | |
66 | */ | |
67 | #define VMW_BALLOON_RATE_ALLOC_MIN 512U | |
68 | #define VMW_BALLOON_RATE_ALLOC_MAX 2048U | |
69 | #define VMW_BALLOON_RATE_ALLOC_INC 16U | |
70 | ||
453dc659 DT |
71 | /* |
72 | * When guest is under memory pressure, use a reduced page allocation | |
73 | * rate for next several cycles. | |
74 | */ | |
75 | #define VMW_BALLOON_SLOW_CYCLES 4 | |
76 | ||
77 | /* | |
78 | * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't | |
71baba4b | 79 | * allow wait (__GFP_RECLAIM) for NOSLEEP page allocations. Use |
453dc659 DT |
80 | * __GFP_NOWARN, to suppress page allocation failure warnings. |
81 | */ | |
82 | #define VMW_PAGE_ALLOC_NOSLEEP (__GFP_HIGHMEM|__GFP_NOWARN) | |
83 | ||
84 | /* | |
85 | * Use GFP_HIGHUSER when executing in a separate kernel thread | |
86 | * context and allocation can sleep. This is less stressful to | |
87 | * the guest memory system, since it allows the thread to block | |
88 | * while memory is reclaimed, and won't take pages from emergency | |
89 | * low-memory pools. | |
90 | */ | |
91 | #define VMW_PAGE_ALLOC_CANSLEEP (GFP_HIGHUSER) | |
92 | ||
55adaa49 DT |
93 | /* Maximum number of refused pages we accumulate during inflation cycle */ |
94 | #define VMW_BALLOON_MAX_REFUSED 16 | |
453dc659 DT |
95 | |
96 | /* | |
97 | * Hypervisor communication port definitions. | |
98 | */ | |
99 | #define VMW_BALLOON_HV_PORT 0x5670 | |
100 | #define VMW_BALLOON_HV_MAGIC 0x456c6d6f | |
453dc659 DT |
101 | #define VMW_BALLOON_GUEST_ID 1 /* Linux */ |
102 | ||
eb79100f XD |
103 | enum vmwballoon_capabilities { |
104 | /* | |
105 | * Bit 0 is reserved and not associated to any capability. | |
106 | */ | |
48e3d668 PM |
107 | VMW_BALLOON_BASIC_CMDS = (1 << 1), |
108 | VMW_BALLOON_BATCHED_CMDS = (1 << 2), | |
109 | VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3), | |
110 | VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4), | |
eb79100f XD |
111 | }; |
112 | ||
f220a80f | 113 | #define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_BASIC_CMDS \ |
365bd7ef | 114 | | VMW_BALLOON_BATCHED_CMDS \ |
48e3d668 PM |
115 | | VMW_BALLOON_BATCHED_2M_CMDS \ |
116 | | VMW_BALLOON_SIGNALLED_WAKEUP_CMD) | |
365bd7ef PM |
117 | |
118 | #define VMW_BALLOON_2M_SHIFT (9) | |
119 | #define VMW_BALLOON_NUM_PAGE_SIZES (2) | |
eb79100f | 120 | |
f220a80f XD |
121 | /* |
122 | * Backdoor commands availability: | |
123 | * | |
124 | * START, GET_TARGET and GUEST_ID are always available, | |
125 | * | |
126 | * VMW_BALLOON_BASIC_CMDS: | |
127 | * LOCK and UNLOCK commands, | |
128 | * VMW_BALLOON_BATCHED_CMDS: | |
129 | * BATCHED_LOCK and BATCHED_UNLOCK commands. | |
365bd7ef | 130 | * VMW BALLOON_BATCHED_2M_CMDS: |
48e3d668 PM |
131 | * BATCHED_2M_LOCK and BATCHED_2M_UNLOCK commands, |
132 | * VMW VMW_BALLOON_SIGNALLED_WAKEUP_CMD: | |
133 | * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command. | |
f220a80f | 134 | */ |
365bd7ef PM |
135 | #define VMW_BALLOON_CMD_START 0 |
136 | #define VMW_BALLOON_CMD_GET_TARGET 1 | |
137 | #define VMW_BALLOON_CMD_LOCK 2 | |
138 | #define VMW_BALLOON_CMD_UNLOCK 3 | |
139 | #define VMW_BALLOON_CMD_GUEST_ID 4 | |
140 | #define VMW_BALLOON_CMD_BATCHED_LOCK 6 | |
141 | #define VMW_BALLOON_CMD_BATCHED_UNLOCK 7 | |
142 | #define VMW_BALLOON_CMD_BATCHED_2M_LOCK 8 | |
143 | #define VMW_BALLOON_CMD_BATCHED_2M_UNLOCK 9 | |
48e3d668 | 144 | #define VMW_BALLOON_CMD_VMCI_DOORBELL_SET 10 |
365bd7ef | 145 | |
453dc659 DT |
146 | |
147 | /* error codes */ | |
eb79100f XD |
148 | #define VMW_BALLOON_SUCCESS 0 |
149 | #define VMW_BALLOON_FAILURE -1 | |
150 | #define VMW_BALLOON_ERROR_CMD_INVALID 1 | |
151 | #define VMW_BALLOON_ERROR_PPN_INVALID 2 | |
152 | #define VMW_BALLOON_ERROR_PPN_LOCKED 3 | |
153 | #define VMW_BALLOON_ERROR_PPN_UNLOCKED 4 | |
154 | #define VMW_BALLOON_ERROR_PPN_PINNED 5 | |
155 | #define VMW_BALLOON_ERROR_PPN_NOTNEEDED 6 | |
156 | #define VMW_BALLOON_ERROR_RESET 7 | |
157 | #define VMW_BALLOON_ERROR_BUSY 8 | |
158 | ||
159 | #define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000) | |
160 | ||
f220a80f XD |
161 | /* Batch page description */ |
162 | ||
163 | /* | |
164 | * Layout of a page in the batch page: | |
165 | * | |
166 | * +-------------+----------+--------+ | |
167 | * | | | | | |
168 | * | Page number | Reserved | Status | | |
169 | * | | | | | |
170 | * +-------------+----------+--------+ | |
171 | * 64 PAGE_SHIFT 6 0 | |
172 | * | |
f220a80f XD |
173 | * The reserved field should be set to 0. |
174 | */ | |
175 | #define VMW_BALLOON_BATCH_MAX_PAGES (PAGE_SIZE / sizeof(u64)) | |
176 | #define VMW_BALLOON_BATCH_STATUS_MASK ((1UL << 5) - 1) | |
177 | #define VMW_BALLOON_BATCH_PAGE_MASK (~((1UL << PAGE_SHIFT) - 1)) | |
178 | ||
179 | struct vmballoon_batch_page { | |
180 | u64 pages[VMW_BALLOON_BATCH_MAX_PAGES]; | |
181 | }; | |
182 | ||
183 | static u64 vmballoon_batch_get_pa(struct vmballoon_batch_page *batch, int idx) | |
184 | { | |
185 | return batch->pages[idx] & VMW_BALLOON_BATCH_PAGE_MASK; | |
186 | } | |
187 | ||
188 | static int vmballoon_batch_get_status(struct vmballoon_batch_page *batch, | |
189 | int idx) | |
190 | { | |
191 | return (int)(batch->pages[idx] & VMW_BALLOON_BATCH_STATUS_MASK); | |
192 | } | |
193 | ||
194 | static void vmballoon_batch_set_pa(struct vmballoon_batch_page *batch, int idx, | |
195 | u64 pa) | |
196 | { | |
197 | batch->pages[idx] = pa; | |
198 | } | |
199 | ||
200 | ||
201 | #define VMWARE_BALLOON_CMD(cmd, arg1, arg2, result) \ | |
eb79100f | 202 | ({ \ |
f220a80f | 203 | unsigned long __status, __dummy1, __dummy2, __dummy3; \ |
eb79100f XD |
204 | __asm__ __volatile__ ("inl %%dx" : \ |
205 | "=a"(__status), \ | |
206 | "=c"(__dummy1), \ | |
207 | "=d"(__dummy2), \ | |
f220a80f XD |
208 | "=b"(result), \ |
209 | "=S" (__dummy3) : \ | |
eb79100f XD |
210 | "0"(VMW_BALLOON_HV_MAGIC), \ |
211 | "1"(VMW_BALLOON_CMD_##cmd), \ | |
212 | "2"(VMW_BALLOON_HV_PORT), \ | |
f220a80f XD |
213 | "3"(arg1), \ |
214 | "4" (arg2) : \ | |
eb79100f XD |
215 | "memory"); \ |
216 | if (VMW_BALLOON_CMD_##cmd == VMW_BALLOON_CMD_START) \ | |
217 | result = __dummy1; \ | |
218 | result &= -1UL; \ | |
219 | __status & -1UL; \ | |
453dc659 DT |
220 | }) |
221 | ||
222 | #ifdef CONFIG_DEBUG_FS | |
223 | struct vmballoon_stats { | |
224 | unsigned int timer; | |
48e3d668 | 225 | unsigned int doorbell; |
453dc659 | 226 | |
2ca02df6 | 227 | /* allocation statistics */ |
365bd7ef PM |
228 | unsigned int alloc[VMW_BALLOON_NUM_PAGE_SIZES]; |
229 | unsigned int alloc_fail[VMW_BALLOON_NUM_PAGE_SIZES]; | |
453dc659 DT |
230 | unsigned int sleep_alloc; |
231 | unsigned int sleep_alloc_fail; | |
365bd7ef PM |
232 | unsigned int refused_alloc[VMW_BALLOON_NUM_PAGE_SIZES]; |
233 | unsigned int refused_free[VMW_BALLOON_NUM_PAGE_SIZES]; | |
234 | unsigned int free[VMW_BALLOON_NUM_PAGE_SIZES]; | |
453dc659 DT |
235 | |
236 | /* monitor operations */ | |
365bd7ef PM |
237 | unsigned int lock[VMW_BALLOON_NUM_PAGE_SIZES]; |
238 | unsigned int lock_fail[VMW_BALLOON_NUM_PAGE_SIZES]; | |
239 | unsigned int unlock[VMW_BALLOON_NUM_PAGE_SIZES]; | |
240 | unsigned int unlock_fail[VMW_BALLOON_NUM_PAGE_SIZES]; | |
453dc659 DT |
241 | unsigned int target; |
242 | unsigned int target_fail; | |
243 | unsigned int start; | |
244 | unsigned int start_fail; | |
245 | unsigned int guest_type; | |
246 | unsigned int guest_type_fail; | |
48e3d668 PM |
247 | unsigned int doorbell_set; |
248 | unsigned int doorbell_unset; | |
453dc659 DT |
249 | }; |
250 | ||
251 | #define STATS_INC(stat) (stat)++ | |
252 | #else | |
253 | #define STATS_INC(stat) | |
254 | #endif | |
255 | ||
f220a80f XD |
256 | struct vmballoon; |
257 | ||
258 | struct vmballoon_ops { | |
259 | void (*add_page)(struct vmballoon *b, int idx, struct page *p); | |
4670de4d | 260 | int (*lock)(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 261 | bool is_2m_pages, unsigned int *target); |
4670de4d | 262 | int (*unlock)(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 263 | bool is_2m_pages, unsigned int *target); |
f220a80f XD |
264 | }; |
265 | ||
365bd7ef | 266 | struct vmballoon_page_size { |
453dc659 DT |
267 | /* list of reserved physical pages */ |
268 | struct list_head pages; | |
269 | ||
270 | /* transient list of non-balloonable pages */ | |
271 | struct list_head refused_pages; | |
55adaa49 | 272 | unsigned int n_refused_pages; |
365bd7ef PM |
273 | }; |
274 | ||
275 | struct vmballoon { | |
276 | struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES]; | |
277 | ||
278 | /* supported page sizes. 1 == 4k pages only, 2 == 4k and 2m pages */ | |
279 | unsigned supported_page_sizes; | |
453dc659 DT |
280 | |
281 | /* balloon size in pages */ | |
282 | unsigned int size; | |
283 | unsigned int target; | |
284 | ||
285 | /* reset flag */ | |
286 | bool reset_required; | |
287 | ||
288 | /* adjustment rates (pages per second) */ | |
289 | unsigned int rate_alloc; | |
453dc659 DT |
290 | |
291 | /* slowdown page allocations for next few cycles */ | |
292 | unsigned int slow_allocation_cycles; | |
293 | ||
f220a80f XD |
294 | unsigned long capabilities; |
295 | ||
296 | struct vmballoon_batch_page *batch_page; | |
297 | unsigned int batch_max_pages; | |
298 | struct page *page; | |
299 | ||
300 | const struct vmballoon_ops *ops; | |
301 | ||
453dc659 DT |
302 | #ifdef CONFIG_DEBUG_FS |
303 | /* statistics */ | |
304 | struct vmballoon_stats stats; | |
305 | ||
306 | /* debugfs file exporting statistics */ | |
307 | struct dentry *dbg_entry; | |
308 | #endif | |
309 | ||
310 | struct sysinfo sysinfo; | |
311 | ||
312 | struct delayed_work dwork; | |
48e3d668 PM |
313 | |
314 | struct vmci_handle vmci_doorbell; | |
453dc659 DT |
315 | }; |
316 | ||
317 | static struct vmballoon balloon; | |
453dc659 DT |
318 | |
319 | /* | |
320 | * Send "start" command to the host, communicating supported version | |
321 | * of the protocol. | |
322 | */ | |
f220a80f | 323 | static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps) |
453dc659 | 324 | { |
f220a80f | 325 | unsigned long status, capabilities, dummy = 0; |
365bd7ef | 326 | bool success; |
453dc659 DT |
327 | |
328 | STATS_INC(b->stats.start); | |
329 | ||
f220a80f XD |
330 | status = VMWARE_BALLOON_CMD(START, req_caps, dummy, capabilities); |
331 | ||
332 | switch (status) { | |
333 | case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES: | |
334 | b->capabilities = capabilities; | |
365bd7ef PM |
335 | success = true; |
336 | break; | |
f220a80f XD |
337 | case VMW_BALLOON_SUCCESS: |
338 | b->capabilities = VMW_BALLOON_BASIC_CMDS; | |
365bd7ef PM |
339 | success = true; |
340 | break; | |
341 | default: | |
342 | success = false; | |
f220a80f | 343 | } |
453dc659 | 344 | |
d3b40384 NA |
345 | /* |
346 | * 2MB pages are only supported with batching. If batching is for some | |
347 | * reason disabled, do not use 2MB pages, since otherwise the legacy | |
348 | * mechanism is used with 2MB pages, causing a failure. | |
349 | */ | |
350 | if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) && | |
351 | (b->capabilities & VMW_BALLOON_BATCHED_CMDS)) | |
365bd7ef PM |
352 | b->supported_page_sizes = 2; |
353 | else | |
354 | b->supported_page_sizes = 1; | |
355 | ||
356 | if (!success) { | |
357 | pr_debug("%s - failed, hv returns %ld\n", __func__, status); | |
358 | STATS_INC(b->stats.start_fail); | |
359 | } | |
360 | return success; | |
453dc659 DT |
361 | } |
362 | ||
363 | static bool vmballoon_check_status(struct vmballoon *b, unsigned long status) | |
364 | { | |
365 | switch (status) { | |
366 | case VMW_BALLOON_SUCCESS: | |
367 | return true; | |
368 | ||
369 | case VMW_BALLOON_ERROR_RESET: | |
370 | b->reset_required = true; | |
371 | /* fall through */ | |
372 | ||
373 | default: | |
374 | return false; | |
375 | } | |
376 | } | |
377 | ||
378 | /* | |
379 | * Communicate guest type to the host so that it can adjust ballooning | |
380 | * algorithm to the one most appropriate for the guest. This command | |
381 | * is normally issued after sending "start" command and is part of | |
382 | * standard reset sequence. | |
383 | */ | |
384 | static bool vmballoon_send_guest_id(struct vmballoon *b) | |
385 | { | |
f220a80f | 386 | unsigned long status, dummy = 0; |
453dc659 | 387 | |
f220a80f XD |
388 | status = VMWARE_BALLOON_CMD(GUEST_ID, VMW_BALLOON_GUEST_ID, dummy, |
389 | dummy); | |
453dc659 DT |
390 | |
391 | STATS_INC(b->stats.guest_type); | |
392 | ||
393 | if (vmballoon_check_status(b, status)) | |
394 | return true; | |
395 | ||
396 | pr_debug("%s - failed, hv returns %ld\n", __func__, status); | |
397 | STATS_INC(b->stats.guest_type_fail); | |
398 | return false; | |
399 | } | |
400 | ||
365bd7ef PM |
401 | static u16 vmballoon_page_size(bool is_2m_page) |
402 | { | |
403 | if (is_2m_page) | |
404 | return 1 << VMW_BALLOON_2M_SHIFT; | |
405 | ||
406 | return 1; | |
407 | } | |
408 | ||
453dc659 DT |
409 | /* |
410 | * Retrieve desired balloon size from the host. | |
411 | */ | |
412 | static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target) | |
413 | { | |
414 | unsigned long status; | |
415 | unsigned long target; | |
416 | unsigned long limit; | |
f220a80f | 417 | unsigned long dummy = 0; |
453dc659 DT |
418 | u32 limit32; |
419 | ||
420 | /* | |
421 | * si_meminfo() is cheap. Moreover, we want to provide dynamic | |
422 | * max balloon size later. So let us call si_meminfo() every | |
423 | * iteration. | |
424 | */ | |
425 | si_meminfo(&b->sysinfo); | |
426 | limit = b->sysinfo.totalram; | |
427 | ||
428 | /* Ensure limit fits in 32-bits */ | |
429 | limit32 = (u32)limit; | |
430 | if (limit != limit32) | |
431 | return false; | |
432 | ||
433 | /* update stats */ | |
434 | STATS_INC(b->stats.target); | |
435 | ||
f220a80f | 436 | status = VMWARE_BALLOON_CMD(GET_TARGET, limit, dummy, target); |
453dc659 DT |
437 | if (vmballoon_check_status(b, status)) { |
438 | *new_target = target; | |
439 | return true; | |
440 | } | |
441 | ||
442 | pr_debug("%s - failed, hv returns %ld\n", __func__, status); | |
443 | STATS_INC(b->stats.target_fail); | |
444 | return false; | |
445 | } | |
446 | ||
447 | /* | |
448 | * Notify the host about allocated page so that host can use it without | |
449 | * fear that guest will need it. Host may reject some pages, we need to | |
450 | * check the return value and maybe submit a different page. | |
451 | */ | |
3e5ba466 | 452 | static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn, |
4670de4d | 453 | unsigned int *hv_status, unsigned int *target) |
453dc659 | 454 | { |
f220a80f | 455 | unsigned long status, dummy = 0; |
453dc659 DT |
456 | u32 pfn32; |
457 | ||
458 | pfn32 = (u32)pfn; | |
459 | if (pfn32 != pfn) | |
9fd44e90 | 460 | return -EINVAL; |
453dc659 | 461 | |
365bd7ef | 462 | STATS_INC(b->stats.lock[false]); |
453dc659 | 463 | |
4670de4d | 464 | *hv_status = status = VMWARE_BALLOON_CMD(LOCK, pfn, dummy, *target); |
453dc659 | 465 | if (vmballoon_check_status(b, status)) |
3e5ba466 | 466 | return 0; |
453dc659 DT |
467 | |
468 | pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 469 | STATS_INC(b->stats.lock_fail[false]); |
9fd44e90 | 470 | return -EIO; |
453dc659 DT |
471 | } |
472 | ||
f220a80f | 473 | static int vmballoon_send_batched_lock(struct vmballoon *b, |
365bd7ef | 474 | unsigned int num_pages, bool is_2m_pages, unsigned int *target) |
f220a80f | 475 | { |
4670de4d | 476 | unsigned long status; |
1e39eb1b | 477 | unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page)); |
f220a80f | 478 | |
365bd7ef PM |
479 | STATS_INC(b->stats.lock[is_2m_pages]); |
480 | ||
481 | if (is_2m_pages) | |
482 | status = VMWARE_BALLOON_CMD(BATCHED_2M_LOCK, pfn, num_pages, | |
483 | *target); | |
484 | else | |
485 | status = VMWARE_BALLOON_CMD(BATCHED_LOCK, pfn, num_pages, | |
486 | *target); | |
f220a80f | 487 | |
f220a80f XD |
488 | if (vmballoon_check_status(b, status)) |
489 | return 0; | |
490 | ||
491 | pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 492 | STATS_INC(b->stats.lock_fail[is_2m_pages]); |
f220a80f XD |
493 | return 1; |
494 | } | |
495 | ||
453dc659 DT |
496 | /* |
497 | * Notify the host that guest intends to release given page back into | |
498 | * the pool of available (to the guest) pages. | |
499 | */ | |
4670de4d XD |
500 | static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn, |
501 | unsigned int *target) | |
453dc659 | 502 | { |
f220a80f | 503 | unsigned long status, dummy = 0; |
453dc659 DT |
504 | u32 pfn32; |
505 | ||
506 | pfn32 = (u32)pfn; | |
507 | if (pfn32 != pfn) | |
508 | return false; | |
509 | ||
365bd7ef | 510 | STATS_INC(b->stats.unlock[false]); |
453dc659 | 511 | |
4670de4d | 512 | status = VMWARE_BALLOON_CMD(UNLOCK, pfn, dummy, *target); |
453dc659 DT |
513 | if (vmballoon_check_status(b, status)) |
514 | return true; | |
515 | ||
516 | pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 517 | STATS_INC(b->stats.unlock_fail[false]); |
453dc659 DT |
518 | return false; |
519 | } | |
520 | ||
f220a80f | 521 | static bool vmballoon_send_batched_unlock(struct vmballoon *b, |
365bd7ef | 522 | unsigned int num_pages, bool is_2m_pages, unsigned int *target) |
f220a80f | 523 | { |
4670de4d | 524 | unsigned long status; |
1e39eb1b | 525 | unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page)); |
f220a80f | 526 | |
365bd7ef PM |
527 | STATS_INC(b->stats.unlock[is_2m_pages]); |
528 | ||
529 | if (is_2m_pages) | |
530 | status = VMWARE_BALLOON_CMD(BATCHED_2M_UNLOCK, pfn, num_pages, | |
531 | *target); | |
532 | else | |
533 | status = VMWARE_BALLOON_CMD(BATCHED_UNLOCK, pfn, num_pages, | |
534 | *target); | |
f220a80f | 535 | |
f220a80f XD |
536 | if (vmballoon_check_status(b, status)) |
537 | return true; | |
538 | ||
539 | pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 540 | STATS_INC(b->stats.unlock_fail[is_2m_pages]); |
f220a80f XD |
541 | return false; |
542 | } | |
543 | ||
365bd7ef PM |
544 | static struct page *vmballoon_alloc_page(gfp_t flags, bool is_2m_page) |
545 | { | |
546 | if (is_2m_page) | |
547 | return alloc_pages(flags, VMW_BALLOON_2M_SHIFT); | |
548 | ||
549 | return alloc_page(flags); | |
550 | } | |
551 | ||
552 | static void vmballoon_free_page(struct page *page, bool is_2m_page) | |
553 | { | |
554 | if (is_2m_page) | |
555 | __free_pages(page, VMW_BALLOON_2M_SHIFT); | |
556 | else | |
557 | __free_page(page); | |
558 | } | |
559 | ||
453dc659 DT |
560 | /* |
561 | * Quickly release all pages allocated for the balloon. This function is | |
562 | * called when host decides to "reset" balloon for one reason or another. | |
563 | * Unlike normal "deflate" we do not (shall not) notify host of the pages | |
564 | * being released. | |
565 | */ | |
566 | static void vmballoon_pop(struct vmballoon *b) | |
567 | { | |
568 | struct page *page, *next; | |
365bd7ef PM |
569 | unsigned is_2m_pages; |
570 | ||
571 | for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES; | |
572 | is_2m_pages++) { | |
573 | struct vmballoon_page_size *page_size = | |
574 | &b->page_sizes[is_2m_pages]; | |
575 | u16 size_per_page = vmballoon_page_size(is_2m_pages); | |
576 | ||
577 | list_for_each_entry_safe(page, next, &page_size->pages, lru) { | |
578 | list_del(&page->lru); | |
579 | vmballoon_free_page(page, is_2m_pages); | |
580 | STATS_INC(b->stats.free[is_2m_pages]); | |
581 | b->size -= size_per_page; | |
582 | cond_resched(); | |
583 | } | |
453dc659 | 584 | } |
453dc659 | 585 | |
5ee296fd GK |
586 | /* Clearing the batch_page unconditionally has no adverse effect */ |
587 | free_page((unsigned long)b->batch_page); | |
588 | b->batch_page = NULL; | |
453dc659 DT |
589 | } |
590 | ||
591 | /* | |
ef0f8f11 XD |
592 | * Notify the host of a ballooned page. If host rejects the page put it on the |
593 | * refuse list, those refused page are then released at the end of the | |
594 | * inflation cycle. | |
453dc659 | 595 | */ |
4670de4d | 596 | static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 597 | bool is_2m_pages, unsigned int *target) |
453dc659 | 598 | { |
ef0f8f11 | 599 | int locked, hv_status; |
f220a80f | 600 | struct page *page = b->page; |
365bd7ef PM |
601 | struct vmballoon_page_size *page_size = &b->page_sizes[false]; |
602 | ||
603 | /* is_2m_pages can never happen as 2m pages support implies batching */ | |
453dc659 | 604 | |
4670de4d XD |
605 | locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status, |
606 | target); | |
9fd44e90 | 607 | if (locked) { |
365bd7ef | 608 | STATS_INC(b->stats.refused_alloc[false]); |
453dc659 | 609 | |
9fd44e90 NA |
610 | if (locked == -EIO && |
611 | (hv_status == VMW_BALLOON_ERROR_RESET || | |
612 | hv_status == VMW_BALLOON_ERROR_PPN_NOTNEEDED)) { | |
365bd7ef | 613 | vmballoon_free_page(page, false); |
ef0f8f11 XD |
614 | return -EIO; |
615 | } | |
453dc659 | 616 | |
ef0f8f11 XD |
617 | /* |
618 | * Place page on the list of non-balloonable pages | |
619 | * and retry allocation, unless we already accumulated | |
620 | * too many of them, in which case take a breather. | |
621 | */ | |
365bd7ef PM |
622 | if (page_size->n_refused_pages < VMW_BALLOON_MAX_REFUSED) { |
623 | page_size->n_refused_pages++; | |
624 | list_add(&page->lru, &page_size->refused_pages); | |
ef0f8f11 | 625 | } else { |
365bd7ef | 626 | vmballoon_free_page(page, false); |
453dc659 | 627 | } |
9fd44e90 | 628 | return locked; |
ef0f8f11 | 629 | } |
453dc659 DT |
630 | |
631 | /* track allocated page */ | |
365bd7ef | 632 | list_add(&page->lru, &page_size->pages); |
453dc659 DT |
633 | |
634 | /* update balloon size */ | |
635 | b->size++; | |
636 | ||
637 | return 0; | |
638 | } | |
639 | ||
f220a80f | 640 | static int vmballoon_lock_batched_page(struct vmballoon *b, |
365bd7ef | 641 | unsigned int num_pages, bool is_2m_pages, unsigned int *target) |
f220a80f XD |
642 | { |
643 | int locked, i; | |
365bd7ef | 644 | u16 size_per_page = vmballoon_page_size(is_2m_pages); |
f220a80f | 645 | |
365bd7ef PM |
646 | locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages, |
647 | target); | |
f220a80f XD |
648 | if (locked > 0) { |
649 | for (i = 0; i < num_pages; i++) { | |
650 | u64 pa = vmballoon_batch_get_pa(b->batch_page, i); | |
651 | struct page *p = pfn_to_page(pa >> PAGE_SHIFT); | |
652 | ||
365bd7ef | 653 | vmballoon_free_page(p, is_2m_pages); |
f220a80f XD |
654 | } |
655 | ||
656 | return -EIO; | |
657 | } | |
658 | ||
659 | for (i = 0; i < num_pages; i++) { | |
660 | u64 pa = vmballoon_batch_get_pa(b->batch_page, i); | |
661 | struct page *p = pfn_to_page(pa >> PAGE_SHIFT); | |
365bd7ef PM |
662 | struct vmballoon_page_size *page_size = |
663 | &b->page_sizes[is_2m_pages]; | |
f220a80f XD |
664 | |
665 | locked = vmballoon_batch_get_status(b->batch_page, i); | |
666 | ||
667 | switch (locked) { | |
668 | case VMW_BALLOON_SUCCESS: | |
365bd7ef PM |
669 | list_add(&p->lru, &page_size->pages); |
670 | b->size += size_per_page; | |
f220a80f XD |
671 | break; |
672 | case VMW_BALLOON_ERROR_PPN_PINNED: | |
673 | case VMW_BALLOON_ERROR_PPN_INVALID: | |
365bd7ef PM |
674 | if (page_size->n_refused_pages |
675 | < VMW_BALLOON_MAX_REFUSED) { | |
676 | list_add(&p->lru, &page_size->refused_pages); | |
677 | page_size->n_refused_pages++; | |
f220a80f XD |
678 | break; |
679 | } | |
680 | /* Fallthrough */ | |
681 | case VMW_BALLOON_ERROR_RESET: | |
682 | case VMW_BALLOON_ERROR_PPN_NOTNEEDED: | |
365bd7ef | 683 | vmballoon_free_page(p, is_2m_pages); |
f220a80f XD |
684 | break; |
685 | default: | |
686 | /* This should never happen */ | |
687 | WARN_ON_ONCE(true); | |
688 | } | |
689 | } | |
690 | ||
691 | return 0; | |
692 | } | |
693 | ||
453dc659 DT |
694 | /* |
695 | * Release the page allocated for the balloon. Note that we first notify | |
696 | * the host so it can make sure the page will be available for the guest | |
697 | * to use, if needed. | |
698 | */ | |
4670de4d | 699 | static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 700 | bool is_2m_pages, unsigned int *target) |
453dc659 | 701 | { |
f220a80f | 702 | struct page *page = b->page; |
365bd7ef PM |
703 | struct vmballoon_page_size *page_size = &b->page_sizes[false]; |
704 | ||
705 | /* is_2m_pages can never happen as 2m pages support implies batching */ | |
453dc659 | 706 | |
4670de4d | 707 | if (!vmballoon_send_unlock_page(b, page_to_pfn(page), target)) { |
365bd7ef | 708 | list_add(&page->lru, &page_size->pages); |
f220a80f XD |
709 | return -EIO; |
710 | } | |
453dc659 DT |
711 | |
712 | /* deallocate page */ | |
365bd7ef PM |
713 | vmballoon_free_page(page, false); |
714 | STATS_INC(b->stats.free[false]); | |
453dc659 DT |
715 | |
716 | /* update balloon size */ | |
717 | b->size--; | |
718 | ||
719 | return 0; | |
720 | } | |
721 | ||
f220a80f | 722 | static int vmballoon_unlock_batched_page(struct vmballoon *b, |
365bd7ef PM |
723 | unsigned int num_pages, bool is_2m_pages, |
724 | unsigned int *target) | |
f220a80f XD |
725 | { |
726 | int locked, i, ret = 0; | |
727 | bool hv_success; | |
365bd7ef | 728 | u16 size_per_page = vmballoon_page_size(is_2m_pages); |
f220a80f | 729 | |
365bd7ef PM |
730 | hv_success = vmballoon_send_batched_unlock(b, num_pages, is_2m_pages, |
731 | target); | |
f220a80f XD |
732 | if (!hv_success) |
733 | ret = -EIO; | |
734 | ||
735 | for (i = 0; i < num_pages; i++) { | |
736 | u64 pa = vmballoon_batch_get_pa(b->batch_page, i); | |
737 | struct page *p = pfn_to_page(pa >> PAGE_SHIFT); | |
365bd7ef PM |
738 | struct vmballoon_page_size *page_size = |
739 | &b->page_sizes[is_2m_pages]; | |
f220a80f XD |
740 | |
741 | locked = vmballoon_batch_get_status(b->batch_page, i); | |
742 | if (!hv_success || locked != VMW_BALLOON_SUCCESS) { | |
743 | /* | |
744 | * That page wasn't successfully unlocked by the | |
745 | * hypervisor, re-add it to the list of pages owned by | |
746 | * the balloon driver. | |
747 | */ | |
365bd7ef | 748 | list_add(&p->lru, &page_size->pages); |
f220a80f XD |
749 | } else { |
750 | /* deallocate page */ | |
365bd7ef PM |
751 | vmballoon_free_page(p, is_2m_pages); |
752 | STATS_INC(b->stats.free[is_2m_pages]); | |
f220a80f XD |
753 | |
754 | /* update balloon size */ | |
365bd7ef | 755 | b->size -= size_per_page; |
f220a80f XD |
756 | } |
757 | } | |
758 | ||
759 | return ret; | |
760 | } | |
761 | ||
453dc659 DT |
762 | /* |
763 | * Release pages that were allocated while attempting to inflate the | |
764 | * balloon but were refused by the host for one reason or another. | |
765 | */ | |
365bd7ef PM |
766 | static void vmballoon_release_refused_pages(struct vmballoon *b, |
767 | bool is_2m_pages) | |
453dc659 DT |
768 | { |
769 | struct page *page, *next; | |
365bd7ef PM |
770 | struct vmballoon_page_size *page_size = |
771 | &b->page_sizes[is_2m_pages]; | |
453dc659 | 772 | |
365bd7ef | 773 | list_for_each_entry_safe(page, next, &page_size->refused_pages, lru) { |
453dc659 | 774 | list_del(&page->lru); |
365bd7ef PM |
775 | vmballoon_free_page(page, is_2m_pages); |
776 | STATS_INC(b->stats.refused_free[is_2m_pages]); | |
453dc659 | 777 | } |
55adaa49 | 778 | |
365bd7ef | 779 | page_size->n_refused_pages = 0; |
453dc659 DT |
780 | } |
781 | ||
f220a80f XD |
782 | static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p) |
783 | { | |
784 | b->page = p; | |
785 | } | |
786 | ||
787 | static void vmballoon_add_batched_page(struct vmballoon *b, int idx, | |
788 | struct page *p) | |
789 | { | |
790 | vmballoon_batch_set_pa(b->batch_page, idx, | |
791 | (u64)page_to_pfn(p) << PAGE_SHIFT); | |
792 | } | |
793 | ||
453dc659 DT |
794 | /* |
795 | * Inflate the balloon towards its target size. Note that we try to limit | |
796 | * the rate of allocation to make sure we are not choking the rest of the | |
797 | * system. | |
798 | */ | |
799 | static void vmballoon_inflate(struct vmballoon *b) | |
800 | { | |
33d268ed | 801 | unsigned rate; |
453dc659 | 802 | unsigned int allocations = 0; |
f220a80f | 803 | unsigned int num_pages = 0; |
453dc659 | 804 | int error = 0; |
ef0f8f11 | 805 | gfp_t flags = VMW_PAGE_ALLOC_NOSLEEP; |
365bd7ef | 806 | bool is_2m_pages; |
453dc659 DT |
807 | |
808 | pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target); | |
809 | ||
810 | /* | |
811 | * First try NOSLEEP page allocations to inflate balloon. | |
812 | * | |
813 | * If we do not throttle nosleep allocations, we can drain all | |
814 | * free pages in the guest quickly (if the balloon target is high). | |
815 | * As a side-effect, draining free pages helps to inform (force) | |
816 | * the guest to start swapping if balloon target is not met yet, | |
817 | * which is a desired behavior. However, balloon driver can consume | |
818 | * all available CPU cycles if too many pages are allocated in a | |
819 | * second. Therefore, we throttle nosleep allocations even when | |
820 | * the guest is not under memory pressure. OTOH, if we have already | |
821 | * predicted that the guest is under memory pressure, then we | |
822 | * slowdown page allocations considerably. | |
823 | */ | |
824 | ||
453dc659 DT |
825 | /* |
826 | * Start with no sleep allocation rate which may be higher | |
827 | * than sleeping allocation rate. | |
828 | */ | |
365bd7ef PM |
829 | if (b->slow_allocation_cycles) { |
830 | rate = b->rate_alloc; | |
831 | is_2m_pages = false; | |
832 | } else { | |
833 | rate = UINT_MAX; | |
834 | is_2m_pages = | |
835 | b->supported_page_sizes == VMW_BALLOON_NUM_PAGE_SIZES; | |
836 | } | |
453dc659 | 837 | |
33d268ed | 838 | pr_debug("%s - goal: %d, no-sleep rate: %u, sleep rate: %d\n", |
4670de4d | 839 | __func__, b->target - b->size, rate, b->rate_alloc); |
453dc659 | 840 | |
33d268ed | 841 | while (!b->reset_required && |
365bd7ef PM |
842 | b->size + num_pages * vmballoon_page_size(is_2m_pages) |
843 | < b->target) { | |
4670de4d | 844 | struct page *page; |
453dc659 | 845 | |
ef0f8f11 | 846 | if (flags == VMW_PAGE_ALLOC_NOSLEEP) |
365bd7ef | 847 | STATS_INC(b->stats.alloc[is_2m_pages]); |
ef0f8f11 XD |
848 | else |
849 | STATS_INC(b->stats.sleep_alloc); | |
453dc659 | 850 | |
365bd7ef | 851 | page = vmballoon_alloc_page(flags, is_2m_pages); |
ef0f8f11 | 852 | if (!page) { |
365bd7ef PM |
853 | STATS_INC(b->stats.alloc_fail[is_2m_pages]); |
854 | ||
855 | if (is_2m_pages) { | |
856 | b->ops->lock(b, num_pages, true, &b->target); | |
857 | ||
858 | /* | |
859 | * ignore errors from locking as we now switch | |
860 | * to 4k pages and we might get different | |
861 | * errors. | |
862 | */ | |
863 | ||
864 | num_pages = 0; | |
865 | is_2m_pages = false; | |
866 | continue; | |
867 | } | |
868 | ||
ef0f8f11 | 869 | if (flags == VMW_PAGE_ALLOC_CANSLEEP) { |
453dc659 DT |
870 | /* |
871 | * CANSLEEP page allocation failed, so guest | |
872 | * is under severe memory pressure. Quickly | |
873 | * decrease allocation rate. | |
874 | */ | |
875 | b->rate_alloc = max(b->rate_alloc / 2, | |
876 | VMW_BALLOON_RATE_ALLOC_MIN); | |
ef0f8f11 | 877 | STATS_INC(b->stats.sleep_alloc_fail); |
453dc659 DT |
878 | break; |
879 | } | |
880 | ||
881 | /* | |
882 | * NOSLEEP page allocation failed, so the guest is | |
883 | * under memory pressure. Let us slow down page | |
884 | * allocations for next few cycles so that the guest | |
885 | * gets out of memory pressure. Also, if we already | |
886 | * allocated b->rate_alloc pages, let's pause, | |
887 | * otherwise switch to sleeping allocations. | |
888 | */ | |
889 | b->slow_allocation_cycles = VMW_BALLOON_SLOW_CYCLES; | |
890 | ||
4670de4d | 891 | if (allocations >= b->rate_alloc) |
453dc659 DT |
892 | break; |
893 | ||
ef0f8f11 | 894 | flags = VMW_PAGE_ALLOC_CANSLEEP; |
453dc659 DT |
895 | /* Lower rate for sleeping allocations. */ |
896 | rate = b->rate_alloc; | |
ef0f8f11 | 897 | continue; |
453dc659 DT |
898 | } |
899 | ||
f220a80f XD |
900 | b->ops->add_page(b, num_pages++, page); |
901 | if (num_pages == b->batch_max_pages) { | |
365bd7ef PM |
902 | error = b->ops->lock(b, num_pages, is_2m_pages, |
903 | &b->target); | |
f220a80f XD |
904 | num_pages = 0; |
905 | if (error) | |
906 | break; | |
907 | } | |
ef0f8f11 | 908 | |
33d268ed | 909 | cond_resched(); |
453dc659 | 910 | |
4670de4d | 911 | if (allocations >= rate) { |
453dc659 DT |
912 | /* We allocated enough pages, let's take a break. */ |
913 | break; | |
914 | } | |
915 | } | |
916 | ||
f220a80f | 917 | if (num_pages > 0) |
365bd7ef | 918 | b->ops->lock(b, num_pages, is_2m_pages, &b->target); |
f220a80f | 919 | |
453dc659 DT |
920 | /* |
921 | * We reached our goal without failures so try increasing | |
922 | * allocation rate. | |
923 | */ | |
4670de4d XD |
924 | if (error == 0 && allocations >= b->rate_alloc) { |
925 | unsigned int mult = allocations / b->rate_alloc; | |
453dc659 DT |
926 | |
927 | b->rate_alloc = | |
928 | min(b->rate_alloc + mult * VMW_BALLOON_RATE_ALLOC_INC, | |
929 | VMW_BALLOON_RATE_ALLOC_MAX); | |
930 | } | |
931 | ||
365bd7ef PM |
932 | vmballoon_release_refused_pages(b, true); |
933 | vmballoon_release_refused_pages(b, false); | |
453dc659 DT |
934 | } |
935 | ||
936 | /* | |
937 | * Decrease the size of the balloon allowing guest to use more memory. | |
938 | */ | |
939 | static void vmballoon_deflate(struct vmballoon *b) | |
940 | { | |
365bd7ef | 941 | unsigned is_2m_pages; |
453dc659 | 942 | |
33d268ed | 943 | pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target); |
453dc659 DT |
944 | |
945 | /* free pages to reach target */ | |
365bd7ef PM |
946 | for (is_2m_pages = 0; is_2m_pages < b->supported_page_sizes; |
947 | is_2m_pages++) { | |
948 | struct page *page, *next; | |
949 | unsigned int num_pages = 0; | |
950 | struct vmballoon_page_size *page_size = | |
951 | &b->page_sizes[is_2m_pages]; | |
952 | ||
953 | list_for_each_entry_safe(page, next, &page_size->pages, lru) { | |
954 | if (b->reset_required || | |
955 | (b->target > 0 && | |
956 | b->size - num_pages | |
957 | * vmballoon_page_size(is_2m_pages) | |
958 | < b->target + vmballoon_page_size(true))) | |
959 | break; | |
f220a80f | 960 | |
365bd7ef PM |
961 | list_del(&page->lru); |
962 | b->ops->add_page(b, num_pages++, page); | |
33d268ed | 963 | |
365bd7ef PM |
964 | if (num_pages == b->batch_max_pages) { |
965 | int error; | |
453dc659 | 966 | |
365bd7ef PM |
967 | error = b->ops->unlock(b, num_pages, |
968 | is_2m_pages, &b->target); | |
969 | num_pages = 0; | |
970 | if (error) | |
971 | return; | |
972 | } | |
33d268ed | 973 | |
365bd7ef PM |
974 | cond_resched(); |
975 | } | |
453dc659 | 976 | |
365bd7ef PM |
977 | if (num_pages > 0) |
978 | b->ops->unlock(b, num_pages, is_2m_pages, &b->target); | |
979 | } | |
f220a80f XD |
980 | } |
981 | ||
982 | static const struct vmballoon_ops vmballoon_basic_ops = { | |
983 | .add_page = vmballoon_add_page, | |
984 | .lock = vmballoon_lock_page, | |
985 | .unlock = vmballoon_unlock_page | |
986 | }; | |
987 | ||
988 | static const struct vmballoon_ops vmballoon_batched_ops = { | |
989 | .add_page = vmballoon_add_batched_page, | |
990 | .lock = vmballoon_lock_batched_page, | |
991 | .unlock = vmballoon_unlock_batched_page | |
992 | }; | |
993 | ||
994 | static bool vmballoon_init_batching(struct vmballoon *b) | |
995 | { | |
5ee296fd | 996 | struct page *page; |
f220a80f | 997 | |
5ee296fd GK |
998 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
999 | if (!page) | |
f220a80f | 1000 | return false; |
f220a80f | 1001 | |
5ee296fd | 1002 | b->batch_page = page_address(page); |
f220a80f XD |
1003 | return true; |
1004 | } | |
1005 | ||
48e3d668 PM |
1006 | /* |
1007 | * Receive notification and resize balloon | |
1008 | */ | |
1009 | static void vmballoon_doorbell(void *client_data) | |
1010 | { | |
1011 | struct vmballoon *b = client_data; | |
1012 | ||
1013 | STATS_INC(b->stats.doorbell); | |
1014 | ||
1015 | mod_delayed_work(system_freezable_wq, &b->dwork, 0); | |
1016 | } | |
1017 | ||
1018 | /* | |
1019 | * Clean up vmci doorbell | |
1020 | */ | |
1021 | static void vmballoon_vmci_cleanup(struct vmballoon *b) | |
1022 | { | |
1023 | int error; | |
1024 | ||
1025 | VMWARE_BALLOON_CMD(VMCI_DOORBELL_SET, VMCI_INVALID_ID, | |
1026 | VMCI_INVALID_ID, error); | |
1027 | STATS_INC(b->stats.doorbell_unset); | |
1028 | ||
1029 | if (!vmci_handle_is_invalid(b->vmci_doorbell)) { | |
1030 | vmci_doorbell_destroy(b->vmci_doorbell); | |
1031 | b->vmci_doorbell = VMCI_INVALID_HANDLE; | |
1032 | } | |
1033 | } | |
1034 | ||
1035 | /* | |
1036 | * Initialize vmci doorbell, to get notified as soon as balloon changes | |
1037 | */ | |
1038 | static int vmballoon_vmci_init(struct vmballoon *b) | |
1039 | { | |
89667b26 | 1040 | unsigned long error, dummy; |
48e3d668 | 1041 | |
89667b26 NA |
1042 | if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0) |
1043 | return 0; | |
48e3d668 | 1044 | |
89667b26 NA |
1045 | error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB, |
1046 | VMCI_PRIVILEGE_FLAG_RESTRICTED, | |
1047 | vmballoon_doorbell, b); | |
48e3d668 | 1048 | |
89667b26 NA |
1049 | if (error != VMCI_SUCCESS) |
1050 | goto fail; | |
1051 | ||
1052 | error = VMWARE_BALLOON_CMD(VMCI_DOORBELL_SET, b->vmci_doorbell.context, | |
1053 | b->vmci_doorbell.resource, dummy); | |
1054 | ||
1055 | STATS_INC(b->stats.doorbell_set); | |
1056 | ||
1057 | if (error != VMW_BALLOON_SUCCESS) | |
1058 | goto fail; | |
48e3d668 PM |
1059 | |
1060 | return 0; | |
89667b26 NA |
1061 | fail: |
1062 | vmballoon_vmci_cleanup(b); | |
1063 | return -EIO; | |
48e3d668 PM |
1064 | } |
1065 | ||
f220a80f XD |
1066 | /* |
1067 | * Perform standard reset sequence by popping the balloon (in case it | |
1068 | * is not empty) and then restarting protocol. This operation normally | |
1069 | * happens when host responds with VMW_BALLOON_ERROR_RESET to a command. | |
1070 | */ | |
1071 | static void vmballoon_reset(struct vmballoon *b) | |
1072 | { | |
48e3d668 PM |
1073 | int error; |
1074 | ||
1075 | vmballoon_vmci_cleanup(b); | |
1076 | ||
f220a80f XD |
1077 | /* free all pages, skipping monitor unlock */ |
1078 | vmballoon_pop(b); | |
1079 | ||
1080 | if (!vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES)) | |
1081 | return; | |
1082 | ||
1083 | if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) { | |
1084 | b->ops = &vmballoon_batched_ops; | |
1085 | b->batch_max_pages = VMW_BALLOON_BATCH_MAX_PAGES; | |
1086 | if (!vmballoon_init_batching(b)) { | |
1087 | /* | |
1088 | * We failed to initialize batching, inform the monitor | |
1089 | * about it by sending a null capability. | |
1090 | * | |
1091 | * The guest will retry in one second. | |
1092 | */ | |
1093 | vmballoon_send_start(b, 0); | |
1094 | return; | |
1095 | } | |
1096 | } else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) { | |
1097 | b->ops = &vmballoon_basic_ops; | |
1098 | b->batch_max_pages = 1; | |
1099 | } | |
1100 | ||
1101 | b->reset_required = false; | |
48e3d668 PM |
1102 | |
1103 | error = vmballoon_vmci_init(b); | |
1104 | if (error) | |
1105 | pr_err("failed to initialize vmci doorbell\n"); | |
1106 | ||
f220a80f XD |
1107 | if (!vmballoon_send_guest_id(b)) |
1108 | pr_err("failed to send guest ID to the host\n"); | |
453dc659 DT |
1109 | } |
1110 | ||
1111 | /* | |
1112 | * Balloon work function: reset protocol, if needed, get the new size and | |
1113 | * adjust balloon as needed. Repeat in 1 sec. | |
1114 | */ | |
1115 | static void vmballoon_work(struct work_struct *work) | |
1116 | { | |
1117 | struct delayed_work *dwork = to_delayed_work(work); | |
1118 | struct vmballoon *b = container_of(dwork, struct vmballoon, dwork); | |
1119 | unsigned int target; | |
1120 | ||
1121 | STATS_INC(b->stats.timer); | |
1122 | ||
1123 | if (b->reset_required) | |
1124 | vmballoon_reset(b); | |
1125 | ||
1126 | if (b->slow_allocation_cycles > 0) | |
1127 | b->slow_allocation_cycles--; | |
1128 | ||
d7568c13 | 1129 | if (!b->reset_required && vmballoon_send_get_target(b, &target)) { |
453dc659 DT |
1130 | /* update target, adjust size */ |
1131 | b->target = target; | |
1132 | ||
1133 | if (b->size < target) | |
1134 | vmballoon_inflate(b); | |
365bd7ef PM |
1135 | else if (target == 0 || |
1136 | b->size > target + vmballoon_page_size(true)) | |
453dc659 DT |
1137 | vmballoon_deflate(b); |
1138 | } | |
1139 | ||
beda94da DT |
1140 | /* |
1141 | * We are using a freezable workqueue so that balloon operations are | |
1142 | * stopped while the system transitions to/from sleep/hibernation. | |
1143 | */ | |
1144 | queue_delayed_work(system_freezable_wq, | |
1145 | dwork, round_jiffies_relative(HZ)); | |
453dc659 DT |
1146 | } |
1147 | ||
1148 | /* | |
1149 | * DEBUGFS Interface | |
1150 | */ | |
1151 | #ifdef CONFIG_DEBUG_FS | |
1152 | ||
1153 | static int vmballoon_debug_show(struct seq_file *f, void *offset) | |
1154 | { | |
1155 | struct vmballoon *b = f->private; | |
1156 | struct vmballoon_stats *stats = &b->stats; | |
1157 | ||
b36e89da PM |
1158 | /* format capabilities info */ |
1159 | seq_printf(f, | |
1160 | "balloon capabilities: %#4x\n" | |
d7568c13 PM |
1161 | "used capabilities: %#4lx\n" |
1162 | "is resetting: %c\n", | |
1163 | VMW_BALLOON_CAPABILITIES, b->capabilities, | |
1164 | b->reset_required ? 'y' : 'n'); | |
b36e89da | 1165 | |
453dc659 DT |
1166 | /* format size info */ |
1167 | seq_printf(f, | |
1168 | "target: %8d pages\n" | |
1169 | "current: %8d pages\n", | |
1170 | b->target, b->size); | |
1171 | ||
1172 | /* format rate info */ | |
1173 | seq_printf(f, | |
33d268ed PM |
1174 | "rateSleepAlloc: %8d pages/sec\n", |
1175 | b->rate_alloc); | |
453dc659 DT |
1176 | |
1177 | seq_printf(f, | |
1178 | "\n" | |
1179 | "timer: %8u\n" | |
48e3d668 | 1180 | "doorbell: %8u\n" |
453dc659 DT |
1181 | "start: %8u (%4u failed)\n" |
1182 | "guestType: %8u (%4u failed)\n" | |
365bd7ef | 1183 | "2m-lock: %8u (%4u failed)\n" |
453dc659 | 1184 | "lock: %8u (%4u failed)\n" |
365bd7ef | 1185 | "2m-unlock: %8u (%4u failed)\n" |
453dc659 DT |
1186 | "unlock: %8u (%4u failed)\n" |
1187 | "target: %8u (%4u failed)\n" | |
365bd7ef | 1188 | "prim2mAlloc: %8u (%4u failed)\n" |
453dc659 DT |
1189 | "primNoSleepAlloc: %8u (%4u failed)\n" |
1190 | "primCanSleepAlloc: %8u (%4u failed)\n" | |
365bd7ef | 1191 | "prim2mFree: %8u\n" |
453dc659 | 1192 | "primFree: %8u\n" |
365bd7ef | 1193 | "err2mAlloc: %8u\n" |
453dc659 | 1194 | "errAlloc: %8u\n" |
365bd7ef | 1195 | "err2mFree: %8u\n" |
48e3d668 PM |
1196 | "errFree: %8u\n" |
1197 | "doorbellSet: %8u\n" | |
1198 | "doorbellUnset: %8u\n", | |
453dc659 | 1199 | stats->timer, |
48e3d668 | 1200 | stats->doorbell, |
453dc659 DT |
1201 | stats->start, stats->start_fail, |
1202 | stats->guest_type, stats->guest_type_fail, | |
365bd7ef PM |
1203 | stats->lock[true], stats->lock_fail[true], |
1204 | stats->lock[false], stats->lock_fail[false], | |
1205 | stats->unlock[true], stats->unlock_fail[true], | |
1206 | stats->unlock[false], stats->unlock_fail[false], | |
453dc659 | 1207 | stats->target, stats->target_fail, |
365bd7ef PM |
1208 | stats->alloc[true], stats->alloc_fail[true], |
1209 | stats->alloc[false], stats->alloc_fail[false], | |
453dc659 | 1210 | stats->sleep_alloc, stats->sleep_alloc_fail, |
365bd7ef PM |
1211 | stats->free[true], |
1212 | stats->free[false], | |
1213 | stats->refused_alloc[true], stats->refused_alloc[false], | |
48e3d668 PM |
1214 | stats->refused_free[true], stats->refused_free[false], |
1215 | stats->doorbell_set, stats->doorbell_unset); | |
453dc659 DT |
1216 | |
1217 | return 0; | |
1218 | } | |
1219 | ||
1220 | static int vmballoon_debug_open(struct inode *inode, struct file *file) | |
1221 | { | |
1222 | return single_open(file, vmballoon_debug_show, inode->i_private); | |
1223 | } | |
1224 | ||
1225 | static const struct file_operations vmballoon_debug_fops = { | |
1226 | .owner = THIS_MODULE, | |
1227 | .open = vmballoon_debug_open, | |
1228 | .read = seq_read, | |
1229 | .llseek = seq_lseek, | |
1230 | .release = single_release, | |
1231 | }; | |
1232 | ||
1233 | static int __init vmballoon_debugfs_init(struct vmballoon *b) | |
1234 | { | |
1235 | int error; | |
1236 | ||
1237 | b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b, | |
1238 | &vmballoon_debug_fops); | |
1239 | if (IS_ERR(b->dbg_entry)) { | |
1240 | error = PTR_ERR(b->dbg_entry); | |
1241 | pr_err("failed to create debugfs entry, error: %d\n", error); | |
1242 | return error; | |
1243 | } | |
1244 | ||
1245 | return 0; | |
1246 | } | |
1247 | ||
1248 | static void __exit vmballoon_debugfs_exit(struct vmballoon *b) | |
1249 | { | |
1250 | debugfs_remove(b->dbg_entry); | |
1251 | } | |
1252 | ||
1253 | #else | |
1254 | ||
1255 | static inline int vmballoon_debugfs_init(struct vmballoon *b) | |
1256 | { | |
1257 | return 0; | |
1258 | } | |
1259 | ||
1260 | static inline void vmballoon_debugfs_exit(struct vmballoon *b) | |
1261 | { | |
1262 | } | |
1263 | ||
1264 | #endif /* CONFIG_DEBUG_FS */ | |
1265 | ||
1266 | static int __init vmballoon_init(void) | |
1267 | { | |
1268 | int error; | |
365bd7ef | 1269 | unsigned is_2m_pages; |
453dc659 DT |
1270 | /* |
1271 | * Check if we are running on VMware's hypervisor and bail out | |
1272 | * if we are not. | |
1273 | */ | |
399bbc9b | 1274 | if (x86_hyper_type != X86_HYPER_VMWARE) |
453dc659 DT |
1275 | return -ENODEV; |
1276 | ||
365bd7ef PM |
1277 | for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES; |
1278 | is_2m_pages++) { | |
1279 | INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].pages); | |
1280 | INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].refused_pages); | |
1281 | } | |
453dc659 DT |
1282 | |
1283 | /* initialize rates */ | |
1284 | balloon.rate_alloc = VMW_BALLOON_RATE_ALLOC_MAX; | |
453dc659 DT |
1285 | |
1286 | INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work); | |
1287 | ||
453dc659 DT |
1288 | error = vmballoon_debugfs_init(&balloon); |
1289 | if (error) | |
beda94da | 1290 | return error; |
453dc659 | 1291 | |
48e3d668 | 1292 | balloon.vmci_doorbell = VMCI_INVALID_HANDLE; |
d7568c13 PM |
1293 | balloon.batch_page = NULL; |
1294 | balloon.page = NULL; | |
1295 | balloon.reset_required = true; | |
1296 | ||
beda94da | 1297 | queue_delayed_work(system_freezable_wq, &balloon.dwork, 0); |
453dc659 DT |
1298 | |
1299 | return 0; | |
453dc659 | 1300 | } |
bbac5374 NA |
1301 | |
1302 | /* | |
1303 | * Using late_initcall() instead of module_init() allows the balloon to use the | |
1304 | * VMCI doorbell even when the balloon is built into the kernel. Otherwise the | |
1305 | * VMCI is probed only after the balloon is initialized. If the balloon is used | |
1306 | * as a module, late_initcall() is equivalent to module_init(). | |
1307 | */ | |
1308 | late_initcall(vmballoon_init); | |
453dc659 DT |
1309 | |
1310 | static void __exit vmballoon_exit(void) | |
1311 | { | |
48e3d668 | 1312 | vmballoon_vmci_cleanup(&balloon); |
453dc659 | 1313 | cancel_delayed_work_sync(&balloon.dwork); |
453dc659 DT |
1314 | |
1315 | vmballoon_debugfs_exit(&balloon); | |
1316 | ||
1317 | /* | |
1318 | * Deallocate all reserved memory, and reset connection with monitor. | |
1319 | * Reset connection before deallocating memory to avoid potential for | |
1320 | * additional spurious resets from guest touching deallocated pages. | |
1321 | */ | |
d7568c13 | 1322 | vmballoon_send_start(&balloon, 0); |
453dc659 DT |
1323 | vmballoon_pop(&balloon); |
1324 | } | |
1325 | module_exit(vmballoon_exit); |