2 * Stress userfaultfd syscall.
4 * Copyright (C) 2015 Red Hat, Inc.
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
9 * This test allocates two virtual areas and bounces the physical
10 * memory across the two virtual areas (from area_src to area_dst)
13 * There are three threads running per CPU:
15 * 1) one per-CPU thread takes a per-page pthread_mutex in a random
16 * page of the area_dst (while the physical page may still be in
17 * area_src), and increments a per-page counter in the same page,
18 * and checks its value against a verification region.
20 * 2) another per-CPU thread handles the userfaults generated by
21 * thread 1 above. userfaultfd blocking reads or poll() modes are
22 * exercised interleaved.
24 * 3) one last per-CPU thread transfers the memory in the background
25 * at maximum bandwidth (if not already transferred by thread
26 * 2). Each cpu thread takes cares of transferring a portion of the
29 * When all threads of type 3 completed the transfer, one bounce is
30 * complete. area_src and area_dst are then swapped. All threads are
31 * respawned and so the bounce is immediately restarted in the
34 * per-CPU threads 1 by triggering userfaults inside
35 * pthread_mutex_lock will also verify the atomicity of the memory
36 * transfer (UFFDIO_COPY).
38 * The program takes two parameters: the amounts of physical memory in
39 * megabytes (MiB) of the area and the number of bounces to execute.
41 * # 100MiB 99999 bounces
42 * ./userfaultfd 100 99999
45 * ./userfaultfd 1000 99
47 * # 10MiB-~6GiB 999 bounces, continue forever unless an error triggers
48 * while ./userfaultfd $[RANDOM % 6000 + 10] 999; do true; done
56 #include <sys/types.h>
64 #include <sys/syscall.h>
65 #include <sys/ioctl.h>
68 #include <linux/userfaultfd.h>
72 #ifdef __NR_userfaultfd
74 static unsigned long nr_cpus
, nr_pages
, nr_pages_per_cpu
, page_size
;
76 #define BOUNCE_RANDOM (1<<0)
77 #define BOUNCE_RACINGFAULTS (1<<1)
78 #define BOUNCE_VERIFY (1<<2)
79 #define BOUNCE_POLL (1<<3)
83 #define TEST_HUGETLB 2
87 /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
88 #define ALARM_INTERVAL_SECS 10
89 static volatile bool test_uffdio_copy_eexist
= true;
90 static volatile bool test_uffdio_zeropage_eexist
= true;
92 static bool map_shared
;
94 static char *huge_fd_off0
;
95 static unsigned long long *count_verify
;
96 static int uffd
, uffd_flags
, finished
, *pipefd
;
97 static char *area_src
, *area_src_alias
, *area_dst
, *area_dst_alias
;
98 static char *zeropage
;
101 /* pthread_mutex_t starts at page offset 0 */
102 #define area_mutex(___area, ___nr) \
103 ((pthread_mutex_t *) ((___area) + (___nr)*page_size))
105 * count is placed in the page after pthread_mutex_t naturally aligned
106 * to avoid non alignment faults on non-x86 archs.
108 #define area_count(___area, ___nr) \
109 ((volatile unsigned long long *) ((unsigned long) \
110 ((___area) + (___nr)*page_size + \
111 sizeof(pthread_mutex_t) + \
112 sizeof(unsigned long long) - 1) & \
113 ~(unsigned long)(sizeof(unsigned long long) \
116 static int anon_release_pages(char *rel_area
)
120 if (madvise(rel_area
, nr_pages
* page_size
, MADV_DONTNEED
)) {
128 static void anon_allocate_area(void **alloc_area
)
130 if (posix_memalign(alloc_area
, page_size
, nr_pages
* page_size
)) {
131 fprintf(stderr
, "out of memory\n");
136 static void noop_alias_mapping(__u64
*start
, size_t len
, unsigned long offset
)
141 static int hugetlb_release_pages(char *rel_area
)
145 if (fallocate(huge_fd
, FALLOC_FL_PUNCH_HOLE
| FALLOC_FL_KEEP_SIZE
,
146 rel_area
== huge_fd_off0
? 0 :
147 nr_pages
* page_size
,
148 nr_pages
* page_size
)) {
157 static void hugetlb_allocate_area(void **alloc_area
)
159 void *area_alias
= NULL
;
160 char **alloc_area_alias
;
161 *alloc_area
= mmap(NULL
, nr_pages
* page_size
, PROT_READ
| PROT_WRITE
,
162 (map_shared
? MAP_SHARED
: MAP_PRIVATE
) |
164 huge_fd
, *alloc_area
== area_src
? 0 :
165 nr_pages
* page_size
);
166 if (*alloc_area
== MAP_FAILED
) {
167 fprintf(stderr
, "mmap of hugetlbfs file failed\n");
172 area_alias
= mmap(NULL
, nr_pages
* page_size
, PROT_READ
| PROT_WRITE
,
173 MAP_SHARED
| MAP_HUGETLB
,
174 huge_fd
, *alloc_area
== area_src
? 0 :
175 nr_pages
* page_size
);
176 if (area_alias
== MAP_FAILED
) {
177 if (munmap(*alloc_area
, nr_pages
* page_size
) < 0)
178 perror("hugetlb munmap"), exit(1);
183 if (*alloc_area
== area_src
) {
184 huge_fd_off0
= *alloc_area
;
185 alloc_area_alias
= &area_src_alias
;
187 alloc_area_alias
= &area_dst_alias
;
190 *alloc_area_alias
= area_alias
;
193 static void hugetlb_alias_mapping(__u64
*start
, size_t len
, unsigned long offset
)
198 * We can't zap just the pagetable with hugetlbfs because
199 * MADV_DONTEED won't work. So exercise -EEXIST on a alias
200 * mapping where the pagetables are not established initially,
201 * this way we'll exercise the -EEXEC at the fs level.
203 *start
= (unsigned long) area_dst_alias
+ offset
;
207 static int shmem_release_pages(char *rel_area
)
211 if (madvise(rel_area
, nr_pages
* page_size
, MADV_REMOVE
)) {
219 static void shmem_allocate_area(void **alloc_area
)
221 *alloc_area
= mmap(NULL
, nr_pages
* page_size
, PROT_READ
| PROT_WRITE
,
222 MAP_ANONYMOUS
| MAP_SHARED
, -1, 0);
223 if (*alloc_area
== MAP_FAILED
) {
224 fprintf(stderr
, "shared memory mmap failed\n");
229 struct uffd_test_ops
{
230 unsigned long expected_ioctls
;
231 void (*allocate_area
)(void **alloc_area
);
232 int (*release_pages
)(char *rel_area
);
233 void (*alias_mapping
)(__u64
*start
, size_t len
, unsigned long offset
);
236 #define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \
237 (1 << _UFFDIO_COPY) | \
238 (1 << _UFFDIO_ZEROPAGE))
240 static struct uffd_test_ops anon_uffd_test_ops
= {
241 .expected_ioctls
= ANON_EXPECTED_IOCTLS
,
242 .allocate_area
= anon_allocate_area
,
243 .release_pages
= anon_release_pages
,
244 .alias_mapping
= noop_alias_mapping
,
247 static struct uffd_test_ops shmem_uffd_test_ops
= {
248 .expected_ioctls
= ANON_EXPECTED_IOCTLS
,
249 .allocate_area
= shmem_allocate_area
,
250 .release_pages
= shmem_release_pages
,
251 .alias_mapping
= noop_alias_mapping
,
254 static struct uffd_test_ops hugetlb_uffd_test_ops
= {
255 .expected_ioctls
= UFFD_API_RANGE_IOCTLS_BASIC
,
256 .allocate_area
= hugetlb_allocate_area
,
257 .release_pages
= hugetlb_release_pages
,
258 .alias_mapping
= hugetlb_alias_mapping
,
261 static struct uffd_test_ops
*uffd_test_ops
;
263 static int my_bcmp(char *str1
, char *str2
, size_t n
)
266 for (i
= 0; i
< n
; i
++)
267 if (str1
[i
] != str2
[i
])
272 static void *locking_thread(void *arg
)
274 unsigned long cpu
= (unsigned long) arg
;
275 struct random_data rand
;
276 unsigned long page_nr
= *(&(page_nr
)); /* uninitialized warning */
278 unsigned long long count
;
283 if (bounces
& BOUNCE_RANDOM
) {
284 seed
= (unsigned int) time(NULL
) - bounces
;
285 if (!(bounces
& BOUNCE_RACINGFAULTS
))
287 bzero(&rand
, sizeof(rand
));
288 bzero(&randstate
, sizeof(randstate
));
289 if (initstate_r(seed
, randstate
, sizeof(randstate
), &rand
))
290 fprintf(stderr
, "srandom_r error\n"), exit(1);
293 if (!(bounces
& BOUNCE_RACINGFAULTS
))
294 page_nr
+= cpu
* nr_pages_per_cpu
;
298 if (bounces
& BOUNCE_RANDOM
) {
299 if (random_r(&rand
, &rand_nr
))
300 fprintf(stderr
, "random_r 1 error\n"), exit(1);
302 if (sizeof(page_nr
) > sizeof(rand_nr
)) {
303 if (random_r(&rand
, &rand_nr
))
304 fprintf(stderr
, "random_r 2 error\n"), exit(1);
305 page_nr
|= (((unsigned long) rand_nr
) << 16) <<
313 if (bounces
& BOUNCE_VERIFY
) {
314 count
= *area_count(area_dst
, page_nr
);
317 "page_nr %lu wrong count %Lu %Lu\n",
319 count_verify
[page_nr
]), exit(1);
323 * We can't use bcmp (or memcmp) because that
324 * returns 0 erroneously if the memory is
325 * changing under it (even if the end of the
326 * page is never changing and always
330 if (!my_bcmp(area_dst
+ page_nr
* page_size
, zeropage
,
333 "my_bcmp page_nr %lu wrong count %Lu %Lu\n",
335 count_verify
[page_nr
]), exit(1);
340 /* uncomment the below line to test with mutex */
341 /* pthread_mutex_lock(area_mutex(area_dst, page_nr)); */
342 while (!bcmp(area_dst
+ page_nr
* page_size
, zeropage
,
348 /* uncomment below line to test with mutex */
349 /* pthread_mutex_unlock(area_mutex(area_dst, page_nr)); */
352 "page_nr %lu all zero thread %lu %p %lu\n",
353 page_nr
, cpu
, area_dst
+ page_nr
* page_size
,
361 pthread_mutex_lock(area_mutex(area_dst
, page_nr
));
362 count
= *area_count(area_dst
, page_nr
);
363 if (count
!= count_verify
[page_nr
]) {
365 "page_nr %lu memory corruption %Lu %Lu\n",
367 count_verify
[page_nr
]), exit(1);
370 *area_count(area_dst
, page_nr
) = count_verify
[page_nr
] = count
;
371 pthread_mutex_unlock(area_mutex(area_dst
, page_nr
));
373 if (time(NULL
) - start
> 1)
375 "userfault too slow %ld "
376 "possible false positive with overcommit\n",
383 static void retry_copy_page(int ufd
, struct uffdio_copy
*uffdio_copy
,
384 unsigned long offset
)
386 uffd_test_ops
->alias_mapping(&uffdio_copy
->dst
,
389 if (ioctl(ufd
, UFFDIO_COPY
, uffdio_copy
)) {
390 /* real retval in ufdio_copy.copy */
391 if (uffdio_copy
->copy
!= -EEXIST
)
392 fprintf(stderr
, "UFFDIO_COPY retry error %Ld\n",
393 uffdio_copy
->copy
), exit(1);
395 fprintf(stderr
, "UFFDIO_COPY retry unexpected %Ld\n",
396 uffdio_copy
->copy
), exit(1);
400 static int copy_page(int ufd
, unsigned long offset
)
402 struct uffdio_copy uffdio_copy
;
404 if (offset
>= nr_pages
* page_size
)
405 fprintf(stderr
, "unexpected offset %lu\n",
407 uffdio_copy
.dst
= (unsigned long) area_dst
+ offset
;
408 uffdio_copy
.src
= (unsigned long) area_src
+ offset
;
409 uffdio_copy
.len
= page_size
;
410 uffdio_copy
.mode
= 0;
411 uffdio_copy
.copy
= 0;
412 if (ioctl(ufd
, UFFDIO_COPY
, &uffdio_copy
)) {
413 /* real retval in ufdio_copy.copy */
414 if (uffdio_copy
.copy
!= -EEXIST
)
415 fprintf(stderr
, "UFFDIO_COPY error %Ld\n",
416 uffdio_copy
.copy
), exit(1);
417 } else if (uffdio_copy
.copy
!= page_size
) {
418 fprintf(stderr
, "UFFDIO_COPY unexpected copy %Ld\n",
419 uffdio_copy
.copy
), exit(1);
421 if (test_uffdio_copy_eexist
) {
422 test_uffdio_copy_eexist
= false;
423 retry_copy_page(ufd
, &uffdio_copy
, offset
);
430 static void *uffd_poll_thread(void *arg
)
432 unsigned long cpu
= (unsigned long) arg
;
433 struct pollfd pollfd
[2];
435 struct uffdio_register uffd_reg
;
437 unsigned long offset
;
439 unsigned long userfaults
= 0;
442 pollfd
[0].events
= POLLIN
;
443 pollfd
[1].fd
= pipefd
[cpu
*2];
444 pollfd
[1].events
= POLLIN
;
447 ret
= poll(pollfd
, 2, -1);
449 fprintf(stderr
, "poll error %d\n", ret
), exit(1);
451 perror("poll"), exit(1);
452 if (pollfd
[1].revents
& POLLIN
) {
453 if (read(pollfd
[1].fd
, &tmp_chr
, 1) != 1)
454 fprintf(stderr
, "read pipefd error\n"),
458 if (!(pollfd
[0].revents
& POLLIN
))
459 fprintf(stderr
, "pollfd[0].revents %d\n",
460 pollfd
[0].revents
), exit(1);
461 ret
= read(uffd
, &msg
, sizeof(msg
));
465 perror("nonblocking read error"), exit(1);
469 fprintf(stderr
, "unexpected msg event %u\n",
472 case UFFD_EVENT_PAGEFAULT
:
473 if (msg
.arg
.pagefault
.flags
& UFFD_PAGEFAULT_FLAG_WRITE
)
474 fprintf(stderr
, "unexpected write fault\n"), exit(1);
475 offset
= (char *)(unsigned long)msg
.arg
.pagefault
.address
-
477 offset
&= ~(page_size
-1);
478 if (copy_page(uffd
, offset
))
481 case UFFD_EVENT_FORK
:
483 uffd
= msg
.arg
.fork
.ufd
;
486 case UFFD_EVENT_REMOVE
:
487 uffd_reg
.range
.start
= msg
.arg
.remove
.start
;
488 uffd_reg
.range
.len
= msg
.arg
.remove
.end
-
489 msg
.arg
.remove
.start
;
490 if (ioctl(uffd
, UFFDIO_UNREGISTER
, &uffd_reg
.range
))
491 fprintf(stderr
, "remove failure\n"), exit(1);
493 case UFFD_EVENT_REMAP
:
494 area_dst
= (char *)(unsigned long)msg
.arg
.remap
.to
;
498 return (void *)userfaults
;
501 pthread_mutex_t uffd_read_mutex
= PTHREAD_MUTEX_INITIALIZER
;
503 static void *uffd_read_thread(void *arg
)
505 unsigned long *this_cpu_userfaults
;
507 unsigned long offset
;
510 this_cpu_userfaults
= (unsigned long *) arg
;
511 *this_cpu_userfaults
= 0;
513 pthread_mutex_unlock(&uffd_read_mutex
);
514 /* from here cancellation is ok */
517 ret
= read(uffd
, &msg
, sizeof(msg
));
518 if (ret
!= sizeof(msg
)) {
520 perror("blocking read error"), exit(1);
522 fprintf(stderr
, "short read\n"), exit(1);
524 if (msg
.event
!= UFFD_EVENT_PAGEFAULT
)
525 fprintf(stderr
, "unexpected msg event %u\n",
527 if (bounces
& BOUNCE_VERIFY
&&
528 msg
.arg
.pagefault
.flags
& UFFD_PAGEFAULT_FLAG_WRITE
)
529 fprintf(stderr
, "unexpected write fault\n"), exit(1);
530 offset
= (char *)(unsigned long)msg
.arg
.pagefault
.address
-
532 offset
&= ~(page_size
-1);
533 if (copy_page(uffd
, offset
))
534 (*this_cpu_userfaults
)++;
539 static void *background_thread(void *arg
)
541 unsigned long cpu
= (unsigned long) arg
;
542 unsigned long page_nr
;
544 for (page_nr
= cpu
* nr_pages_per_cpu
;
545 page_nr
< (cpu
+1) * nr_pages_per_cpu
;
547 copy_page(uffd
, page_nr
* page_size
);
552 static int stress(unsigned long *userfaults
)
555 pthread_t locking_threads
[nr_cpus
];
556 pthread_t uffd_threads
[nr_cpus
];
557 pthread_t background_threads
[nr_cpus
];
558 void **_userfaults
= (void **) userfaults
;
561 for (cpu
= 0; cpu
< nr_cpus
; cpu
++) {
562 if (pthread_create(&locking_threads
[cpu
], &attr
,
563 locking_thread
, (void *)cpu
))
565 if (bounces
& BOUNCE_POLL
) {
566 if (pthread_create(&uffd_threads
[cpu
], &attr
,
567 uffd_poll_thread
, (void *)cpu
))
570 if (pthread_create(&uffd_threads
[cpu
], &attr
,
574 pthread_mutex_lock(&uffd_read_mutex
);
576 if (pthread_create(&background_threads
[cpu
], &attr
,
577 background_thread
, (void *)cpu
))
580 for (cpu
= 0; cpu
< nr_cpus
; cpu
++)
581 if (pthread_join(background_threads
[cpu
], NULL
))
585 * Be strict and immediately zap area_src, the whole area has
586 * been transferred already by the background treads. The
587 * area_src could then be faulted in in a racy way by still
588 * running uffdio_threads reading zeropages after we zapped
589 * area_src (but they're guaranteed to get -EEXIST from
590 * UFFDIO_COPY without writing zero pages into area_dst
591 * because the background threads already completed).
593 if (uffd_test_ops
->release_pages(area_src
))
596 for (cpu
= 0; cpu
< nr_cpus
; cpu
++) {
598 if (bounces
& BOUNCE_POLL
) {
599 if (write(pipefd
[cpu
*2+1], &c
, 1) != 1) {
600 fprintf(stderr
, "pipefd write error\n");
603 if (pthread_join(uffd_threads
[cpu
], &_userfaults
[cpu
]))
606 if (pthread_cancel(uffd_threads
[cpu
]))
608 if (pthread_join(uffd_threads
[cpu
], NULL
))
614 for (cpu
= 0; cpu
< nr_cpus
; cpu
++)
615 if (pthread_join(locking_threads
[cpu
], NULL
))
621 static int userfaultfd_open(int features
)
623 struct uffdio_api uffdio_api
;
625 uffd
= syscall(__NR_userfaultfd
, O_CLOEXEC
| O_NONBLOCK
);
628 "userfaultfd syscall not available in this kernel\n");
631 uffd_flags
= fcntl(uffd
, F_GETFD
, NULL
);
633 uffdio_api
.api
= UFFD_API
;
634 uffdio_api
.features
= features
;
635 if (ioctl(uffd
, UFFDIO_API
, &uffdio_api
)) {
636 fprintf(stderr
, "UFFDIO_API\n");
639 if (uffdio_api
.api
!= UFFD_API
) {
640 fprintf(stderr
, "UFFDIO_API error %Lu\n", uffdio_api
.api
);
647 sigjmp_buf jbuf
, *sigbuf
;
649 static void sighndl(int sig
, siginfo_t
*siginfo
, void *ptr
)
653 siglongjmp(*sigbuf
, 1);
659 * For non-cooperative userfaultfd test we fork() a process that will
660 * generate pagefaults, will mremap the area monitored by the
661 * userfaultfd and at last this process will release the monitored
663 * For the anonymous and shared memory the area is divided into two
664 * parts, the first part is accessed before mremap, and the second
665 * part is accessed after mremap. Since hugetlbfs does not support
666 * mremap, the entire monitored area is accessed in a single pass for
668 * The release of the pages currently generates event for shmem and
669 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
671 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
672 * monitored area, generate pagefaults and test that signal is delivered.
673 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
674 * test robustness use case - we release monitored area, fork a process
675 * that will generate pagefaults and verify signal is generated.
676 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
677 * feature. Using monitor thread, verify no userfault events are generated.
679 static int faulting_process(int signal_test
)
682 unsigned long long count
;
683 unsigned long split_nr_pages
;
684 unsigned long lastnr
;
685 struct sigaction act
;
686 unsigned long signalled
= 0;
688 if (test_type
!= TEST_HUGETLB
)
689 split_nr_pages
= (nr_pages
+ 1) / 2;
691 split_nr_pages
= nr_pages
;
695 memset(&act
, 0, sizeof(act
));
696 act
.sa_sigaction
= sighndl
;
697 act
.sa_flags
= SA_SIGINFO
;
698 if (sigaction(SIGBUS
, &act
, 0)) {
702 lastnr
= (unsigned long)-1;
705 for (nr
= 0; nr
< split_nr_pages
; nr
++) {
707 if (sigsetjmp(*sigbuf
, 1) != 0) {
709 fprintf(stderr
, "Signal repeated\n");
714 if (signal_test
== 1) {
715 if (copy_page(uffd
, nr
* page_size
))
724 count
= *area_count(area_dst
, nr
);
725 if (count
!= count_verify
[nr
]) {
727 "nr %lu memory corruption %Lu %Lu\n",
729 count_verify
[nr
]), exit(1);
734 return signalled
!= split_nr_pages
;
736 if (test_type
== TEST_HUGETLB
)
739 area_dst
= mremap(area_dst
, nr_pages
* page_size
, nr_pages
* page_size
,
740 MREMAP_MAYMOVE
| MREMAP_FIXED
, area_src
);
741 if (area_dst
== MAP_FAILED
)
742 perror("mremap"), exit(1);
744 for (; nr
< nr_pages
; nr
++) {
745 count
= *area_count(area_dst
, nr
);
746 if (count
!= count_verify
[nr
]) {
748 "nr %lu memory corruption %Lu %Lu\n",
750 count_verify
[nr
]), exit(1);
754 if (uffd_test_ops
->release_pages(area_dst
))
757 for (nr
= 0; nr
< nr_pages
; nr
++) {
758 if (my_bcmp(area_dst
+ nr
* page_size
, zeropage
, page_size
))
759 fprintf(stderr
, "nr %lu is not zero\n", nr
), exit(1);
765 static void retry_uffdio_zeropage(int ufd
,
766 struct uffdio_zeropage
*uffdio_zeropage
,
767 unsigned long offset
)
769 uffd_test_ops
->alias_mapping(&uffdio_zeropage
->range
.start
,
770 uffdio_zeropage
->range
.len
,
772 if (ioctl(ufd
, UFFDIO_ZEROPAGE
, uffdio_zeropage
)) {
773 if (uffdio_zeropage
->zeropage
!= -EEXIST
)
774 fprintf(stderr
, "UFFDIO_ZEROPAGE retry error %Ld\n",
775 uffdio_zeropage
->zeropage
), exit(1);
777 fprintf(stderr
, "UFFDIO_ZEROPAGE retry unexpected %Ld\n",
778 uffdio_zeropage
->zeropage
), exit(1);
782 static int uffdio_zeropage(int ufd
, unsigned long offset
)
784 struct uffdio_zeropage uffdio_zeropage
;
786 unsigned long has_zeropage
;
788 has_zeropage
= uffd_test_ops
->expected_ioctls
& (1 << _UFFDIO_ZEROPAGE
);
790 if (offset
>= nr_pages
* page_size
)
791 fprintf(stderr
, "unexpected offset %lu\n",
793 uffdio_zeropage
.range
.start
= (unsigned long) area_dst
+ offset
;
794 uffdio_zeropage
.range
.len
= page_size
;
795 uffdio_zeropage
.mode
= 0;
796 ret
= ioctl(ufd
, UFFDIO_ZEROPAGE
, &uffdio_zeropage
);
798 /* real retval in ufdio_zeropage.zeropage */
800 if (uffdio_zeropage
.zeropage
== -EEXIST
)
801 fprintf(stderr
, "UFFDIO_ZEROPAGE -EEXIST\n"),
804 fprintf(stderr
, "UFFDIO_ZEROPAGE error %Ld\n",
805 uffdio_zeropage
.zeropage
), exit(1);
807 if (uffdio_zeropage
.zeropage
!= -EINVAL
)
809 "UFFDIO_ZEROPAGE not -EINVAL %Ld\n",
810 uffdio_zeropage
.zeropage
), exit(1);
812 } else if (has_zeropage
) {
813 if (uffdio_zeropage
.zeropage
!= page_size
) {
814 fprintf(stderr
, "UFFDIO_ZEROPAGE unexpected %Ld\n",
815 uffdio_zeropage
.zeropage
), exit(1);
817 if (test_uffdio_zeropage_eexist
) {
818 test_uffdio_zeropage_eexist
= false;
819 retry_uffdio_zeropage(ufd
, &uffdio_zeropage
,
826 "UFFDIO_ZEROPAGE succeeded %Ld\n",
827 uffdio_zeropage
.zeropage
), exit(1);
833 /* exercise UFFDIO_ZEROPAGE */
834 static int userfaultfd_zeropage_test(void)
836 struct uffdio_register uffdio_register
;
837 unsigned long expected_ioctls
;
839 printf("testing UFFDIO_ZEROPAGE: ");
842 if (uffd_test_ops
->release_pages(area_dst
))
845 if (userfaultfd_open(0) < 0)
847 uffdio_register
.range
.start
= (unsigned long) area_dst
;
848 uffdio_register
.range
.len
= nr_pages
* page_size
;
849 uffdio_register
.mode
= UFFDIO_REGISTER_MODE_MISSING
;
850 if (ioctl(uffd
, UFFDIO_REGISTER
, &uffdio_register
))
851 fprintf(stderr
, "register failure\n"), exit(1);
853 expected_ioctls
= uffd_test_ops
->expected_ioctls
;
854 if ((uffdio_register
.ioctls
& expected_ioctls
) !=
857 "unexpected missing ioctl for anon memory\n"),
860 if (uffdio_zeropage(uffd
, 0)) {
861 if (my_bcmp(area_dst
, zeropage
, page_size
))
862 fprintf(stderr
, "zeropage is not zero\n"), exit(1);
870 static int userfaultfd_events_test(void)
872 struct uffdio_register uffdio_register
;
873 unsigned long expected_ioctls
;
874 unsigned long userfaults
;
880 printf("testing events (fork, remap, remove): ");
883 if (uffd_test_ops
->release_pages(area_dst
))
886 features
= UFFD_FEATURE_EVENT_FORK
| UFFD_FEATURE_EVENT_REMAP
|
887 UFFD_FEATURE_EVENT_REMOVE
;
888 if (userfaultfd_open(features
) < 0)
890 fcntl(uffd
, F_SETFL
, uffd_flags
| O_NONBLOCK
);
892 uffdio_register
.range
.start
= (unsigned long) area_dst
;
893 uffdio_register
.range
.len
= nr_pages
* page_size
;
894 uffdio_register
.mode
= UFFDIO_REGISTER_MODE_MISSING
;
895 if (ioctl(uffd
, UFFDIO_REGISTER
, &uffdio_register
))
896 fprintf(stderr
, "register failure\n"), exit(1);
898 expected_ioctls
= uffd_test_ops
->expected_ioctls
;
899 if ((uffdio_register
.ioctls
& expected_ioctls
) !=
902 "unexpected missing ioctl for anon memory\n"),
905 if (pthread_create(&uffd_mon
, &attr
, uffd_poll_thread
, NULL
))
906 perror("uffd_poll_thread create"), exit(1);
910 perror("fork"), exit(1);
913 return faulting_process(0);
915 waitpid(pid
, &err
, 0);
917 fprintf(stderr
, "faulting process failed\n"), exit(1);
919 if (write(pipefd
[1], &c
, sizeof(c
)) != sizeof(c
))
920 perror("pipe write"), exit(1);
921 if (pthread_join(uffd_mon
, (void **)&userfaults
))
925 printf("userfaults: %ld\n", userfaults
);
927 return userfaults
!= nr_pages
;
930 static int userfaultfd_sig_test(void)
932 struct uffdio_register uffdio_register
;
933 unsigned long expected_ioctls
;
934 unsigned long userfaults
;
940 printf("testing signal delivery: ");
943 if (uffd_test_ops
->release_pages(area_dst
))
946 features
= UFFD_FEATURE_EVENT_FORK
|UFFD_FEATURE_SIGBUS
;
947 if (userfaultfd_open(features
) < 0)
949 fcntl(uffd
, F_SETFL
, uffd_flags
| O_NONBLOCK
);
951 uffdio_register
.range
.start
= (unsigned long) area_dst
;
952 uffdio_register
.range
.len
= nr_pages
* page_size
;
953 uffdio_register
.mode
= UFFDIO_REGISTER_MODE_MISSING
;
954 if (ioctl(uffd
, UFFDIO_REGISTER
, &uffdio_register
))
955 fprintf(stderr
, "register failure\n"), exit(1);
957 expected_ioctls
= uffd_test_ops
->expected_ioctls
;
958 if ((uffdio_register
.ioctls
& expected_ioctls
) !=
961 "unexpected missing ioctl for anon memory\n"),
964 if (faulting_process(1))
965 fprintf(stderr
, "faulting process failed\n"), exit(1);
967 if (uffd_test_ops
->release_pages(area_dst
))
970 if (pthread_create(&uffd_mon
, &attr
, uffd_poll_thread
, NULL
))
971 perror("uffd_poll_thread create"), exit(1);
975 perror("fork"), exit(1);
978 exit(faulting_process(2));
980 waitpid(pid
, &err
, 0);
982 fprintf(stderr
, "faulting process failed\n"), exit(1);
984 if (write(pipefd
[1], &c
, sizeof(c
)) != sizeof(c
))
985 perror("pipe write"), exit(1);
986 if (pthread_join(uffd_mon
, (void **)&userfaults
))
990 printf(" Signal test userfaults: %ld\n", userfaults
);
992 return userfaults
!= 0;
994 static int userfaultfd_stress(void)
999 struct uffdio_register uffdio_register
;
1002 unsigned long userfaults
[nr_cpus
];
1004 uffd_test_ops
->allocate_area((void **)&area_src
);
1007 uffd_test_ops
->allocate_area((void **)&area_dst
);
1011 if (userfaultfd_open(0) < 0)
1014 count_verify
= malloc(nr_pages
* sizeof(unsigned long long));
1015 if (!count_verify
) {
1016 perror("count_verify");
1020 for (nr
= 0; nr
< nr_pages
; nr
++) {
1021 *area_mutex(area_src
, nr
) = (pthread_mutex_t
)
1022 PTHREAD_MUTEX_INITIALIZER
;
1023 count_verify
[nr
] = *area_count(area_src
, nr
) = 1;
1025 * In the transition between 255 to 256, powerpc will
1026 * read out of order in my_bcmp and see both bytes as
1027 * zero, so leave a placeholder below always non-zero
1028 * after the count, to avoid my_bcmp to trigger false
1031 *(area_count(area_src
, nr
) + 1) = 1;
1034 pipefd
= malloc(sizeof(int) * nr_cpus
* 2);
1039 for (cpu
= 0; cpu
< nr_cpus
; cpu
++) {
1040 if (pipe2(&pipefd
[cpu
*2], O_CLOEXEC
| O_NONBLOCK
)) {
1046 if (posix_memalign(&area
, page_size
, page_size
)) {
1047 fprintf(stderr
, "out of memory\n");
1051 bzero(zeropage
, page_size
);
1053 pthread_mutex_lock(&uffd_read_mutex
);
1055 pthread_attr_init(&attr
);
1056 pthread_attr_setstacksize(&attr
, 16*1024*1024);
1060 unsigned long expected_ioctls
;
1062 printf("bounces: %d, mode:", bounces
);
1063 if (bounces
& BOUNCE_RANDOM
)
1065 if (bounces
& BOUNCE_RACINGFAULTS
)
1067 if (bounces
& BOUNCE_VERIFY
)
1069 if (bounces
& BOUNCE_POLL
)
1074 if (bounces
& BOUNCE_POLL
)
1075 fcntl(uffd
, F_SETFL
, uffd_flags
| O_NONBLOCK
);
1077 fcntl(uffd
, F_SETFL
, uffd_flags
& ~O_NONBLOCK
);
1080 uffdio_register
.range
.start
= (unsigned long) area_dst
;
1081 uffdio_register
.range
.len
= nr_pages
* page_size
;
1082 uffdio_register
.mode
= UFFDIO_REGISTER_MODE_MISSING
;
1083 if (ioctl(uffd
, UFFDIO_REGISTER
, &uffdio_register
)) {
1084 fprintf(stderr
, "register failure\n");
1087 expected_ioctls
= uffd_test_ops
->expected_ioctls
;
1088 if ((uffdio_register
.ioctls
& expected_ioctls
) !=
1091 "unexpected missing ioctl for anon memory\n");
1095 if (area_dst_alias
) {
1096 uffdio_register
.range
.start
= (unsigned long)
1098 if (ioctl(uffd
, UFFDIO_REGISTER
, &uffdio_register
)) {
1099 fprintf(stderr
, "register failure alias\n");
1105 * The madvise done previously isn't enough: some
1106 * uffd_thread could have read userfaults (one of
1107 * those already resolved by the background thread)
1108 * and it may be in the process of calling
1109 * UFFDIO_COPY. UFFDIO_COPY will read the zapped
1110 * area_src and it would map a zero page in it (of
1111 * course such a UFFDIO_COPY is perfectly safe as it'd
1112 * return -EEXIST). The problem comes at the next
1113 * bounce though: that racing UFFDIO_COPY would
1114 * generate zeropages in the area_src, so invalidating
1115 * the previous MADV_DONTNEED. Without this additional
1116 * MADV_DONTNEED those zeropages leftovers in the
1117 * area_src would lead to -EEXIST failure during the
1118 * next bounce, effectively leaving a zeropage in the
1121 * Try to comment this out madvise to see the memory
1122 * corruption being caught pretty quick.
1124 * khugepaged is also inhibited to collapse THP after
1125 * MADV_DONTNEED only after the UFFDIO_REGISTER, so it's
1126 * required to MADV_DONTNEED here.
1128 if (uffd_test_ops
->release_pages(area_dst
))
1132 if (stress(userfaults
))
1136 if (ioctl(uffd
, UFFDIO_UNREGISTER
, &uffdio_register
.range
)) {
1137 fprintf(stderr
, "unregister failure\n");
1140 if (area_dst_alias
) {
1141 uffdio_register
.range
.start
= (unsigned long) area_dst
;
1142 if (ioctl(uffd
, UFFDIO_UNREGISTER
,
1143 &uffdio_register
.range
)) {
1144 fprintf(stderr
, "unregister failure alias\n");
1150 if (bounces
& BOUNCE_VERIFY
) {
1151 for (nr
= 0; nr
< nr_pages
; nr
++) {
1152 if (*area_count(area_dst
, nr
) != count_verify
[nr
]) {
1154 "error area_count %Lu %Lu %lu\n",
1155 *area_count(area_src
, nr
),
1164 /* prepare next bounce */
1165 tmp_area
= area_src
;
1166 area_src
= area_dst
;
1167 area_dst
= tmp_area
;
1169 tmp_area
= area_src_alias
;
1170 area_src_alias
= area_dst_alias
;
1171 area_dst_alias
= tmp_area
;
1173 printf("userfaults:");
1174 for (cpu
= 0; cpu
< nr_cpus
; cpu
++)
1175 printf(" %lu", userfaults
[cpu
]);
1183 return userfaultfd_zeropage_test() || userfaultfd_sig_test()
1184 || userfaultfd_events_test();
1188 * Copied from mlock2-tests.c
1190 unsigned long default_huge_page_size(void)
1192 unsigned long hps
= 0;
1195 FILE *f
= fopen("/proc/meminfo", "r");
1199 while (getline(&line
, &linelen
, f
) > 0) {
1200 if (sscanf(line
, "Hugepagesize: %lu kB", &hps
) == 1) {
1211 static void set_test_type(const char *type
)
1213 if (!strcmp(type
, "anon")) {
1214 test_type
= TEST_ANON
;
1215 uffd_test_ops
= &anon_uffd_test_ops
;
1216 } else if (!strcmp(type
, "hugetlb")) {
1217 test_type
= TEST_HUGETLB
;
1218 uffd_test_ops
= &hugetlb_uffd_test_ops
;
1219 } else if (!strcmp(type
, "hugetlb_shared")) {
1221 test_type
= TEST_HUGETLB
;
1222 uffd_test_ops
= &hugetlb_uffd_test_ops
;
1223 } else if (!strcmp(type
, "shmem")) {
1225 test_type
= TEST_SHMEM
;
1226 uffd_test_ops
= &shmem_uffd_test_ops
;
1228 fprintf(stderr
, "Unknown test type: %s\n", type
), exit(1);
1231 if (test_type
== TEST_HUGETLB
)
1232 page_size
= default_huge_page_size();
1234 page_size
= sysconf(_SC_PAGE_SIZE
);
1237 fprintf(stderr
, "Unable to determine page size\n"),
1239 if ((unsigned long) area_count(NULL
, 0) + sizeof(unsigned long long) * 2
1241 fprintf(stderr
, "Impossible to run this test\n"), exit(2);
1244 static void sigalrm(int sig
)
1248 test_uffdio_copy_eexist
= true;
1249 test_uffdio_zeropage_eexist
= true;
1250 alarm(ALARM_INTERVAL_SECS
);
1253 int main(int argc
, char **argv
)
1256 fprintf(stderr
, "Usage: <test type> <MiB> <bounces> [hugetlbfs_file]\n"),
1259 if (signal(SIGALRM
, sigalrm
) == SIG_ERR
)
1260 fprintf(stderr
, "failed to arm SIGALRM"), exit(1);
1261 alarm(ALARM_INTERVAL_SECS
);
1263 set_test_type(argv
[1]);
1265 nr_cpus
= sysconf(_SC_NPROCESSORS_ONLN
);
1266 nr_pages_per_cpu
= atol(argv
[2]) * 1024*1024 / page_size
/
1268 if (!nr_pages_per_cpu
) {
1269 fprintf(stderr
, "invalid MiB\n");
1270 fprintf(stderr
, "Usage: <MiB> <bounces>\n"), exit(1);
1273 bounces
= atoi(argv
[3]);
1275 fprintf(stderr
, "invalid bounces\n");
1276 fprintf(stderr
, "Usage: <MiB> <bounces>\n"), exit(1);
1278 nr_pages
= nr_pages_per_cpu
* nr_cpus
;
1280 if (test_type
== TEST_HUGETLB
) {
1282 fprintf(stderr
, "Usage: hugetlb <MiB> <bounces> <hugetlbfs_file>\n"),
1284 huge_fd
= open(argv
[4], O_CREAT
| O_RDWR
, 0755);
1286 fprintf(stderr
, "Open of %s failed", argv
[3]);
1290 if (ftruncate(huge_fd
, 0)) {
1291 fprintf(stderr
, "ftruncate %s to size 0 failed", argv
[3]);
1292 perror("ftruncate");
1296 printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
1297 nr_pages
, nr_pages_per_cpu
);
1298 return userfaultfd_stress();
1301 #else /* __NR_userfaultfd */
1303 #warning "missing __NR_userfaultfd definition"
1307 printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
1311 #endif /* __NR_userfaultfd */