Commit | Line | Data |
---|---|---|
8025e5dd JK |
1 | #include <linux/kernel.h> |
2 | #include <linux/errno.h> | |
3 | #include <linux/err.h> | |
4 | #include <linux/mm.h> | |
5 | #include <linux/slab.h> | |
6 | #include <linux/vmalloc.h> | |
7 | #include <linux/pagemap.h> | |
8 | #include <linux/sched.h> | |
9 | ||
61f9ec1d | 10 | /** |
8025e5dd JK |
11 | * get_vaddr_frames() - map virtual addresses to pfns |
12 | * @start: starting user address | |
13 | * @nr_frames: number of pages / pfns from start to map | |
14 | * @write: whether pages will be written to by the caller | |
15 | * @force: whether to force write access even if user mapping is | |
16 | * readonly. See description of the same argument of | |
17 | get_user_pages(). | |
18 | * @vec: structure which receives pages / pfns of the addresses mapped. | |
19 | * It should have space for at least nr_frames entries. | |
20 | * | |
21 | * This function maps virtual addresses from @start and fills @vec structure | |
22 | * with page frame numbers or page pointers to corresponding pages (choice | |
23 | * depends on the type of the vma underlying the virtual address). If @start | |
24 | * belongs to a normal vma, the function grabs reference to each of the pages | |
25 | * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't | |
26 | * touch page structures and the caller must make sure pfns aren't reused for | |
27 | * anything else while he is using them. | |
28 | * | |
29 | * The function returns number of pages mapped which may be less than | |
30 | * @nr_frames. In particular we stop mapping if there are more vmas of | |
31 | * different type underlying the specified range of virtual addresses. | |
32 | * When the function isn't able to map a single page, it returns error. | |
33 | * | |
34 | * This function takes care of grabbing mmap_sem as necessary. | |
35 | */ | |
36 | int get_vaddr_frames(unsigned long start, unsigned int nr_frames, | |
37 | bool write, bool force, struct frame_vector *vec) | |
38 | { | |
39 | struct mm_struct *mm = current->mm; | |
40 | struct vm_area_struct *vma; | |
41 | int ret = 0; | |
42 | int err; | |
43 | int locked; | |
3b913179 | 44 | unsigned int gup_flags = 0; |
8025e5dd JK |
45 | |
46 | if (nr_frames == 0) | |
47 | return 0; | |
48 | ||
3b913179 LS |
49 | if (write) |
50 | gup_flags |= FOLL_WRITE; | |
51 | if (force) | |
52 | gup_flags |= FOLL_FORCE; | |
53 | ||
8025e5dd JK |
54 | if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) |
55 | nr_frames = vec->nr_allocated; | |
56 | ||
57 | down_read(&mm->mmap_sem); | |
58 | locked = 1; | |
59 | vma = find_vma_intersection(mm, start, start + 1); | |
60 | if (!vma) { | |
61 | ret = -EFAULT; | |
62 | goto out; | |
63 | } | |
64 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) { | |
65 | vec->got_ref = true; | |
66 | vec->is_pfns = false; | |
d4edcf0d | 67 | ret = get_user_pages_locked(start, nr_frames, |
3b913179 | 68 | gup_flags, (struct page **)(vec->ptrs), &locked); |
8025e5dd JK |
69 | goto out; |
70 | } | |
71 | ||
72 | vec->got_ref = false; | |
73 | vec->is_pfns = true; | |
74 | do { | |
75 | unsigned long *nums = frame_vector_pfns(vec); | |
76 | ||
77 | while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { | |
78 | err = follow_pfn(vma, start, &nums[ret]); | |
79 | if (err) { | |
80 | if (ret == 0) | |
81 | ret = err; | |
82 | goto out; | |
83 | } | |
84 | start += PAGE_SIZE; | |
85 | ret++; | |
86 | } | |
87 | /* | |
88 | * We stop if we have enough pages or if VMA doesn't completely | |
89 | * cover the tail page. | |
90 | */ | |
91 | if (ret >= nr_frames || start < vma->vm_end) | |
92 | break; | |
93 | vma = find_vma_intersection(mm, start, start + 1); | |
94 | } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP)); | |
95 | out: | |
96 | if (locked) | |
97 | up_read(&mm->mmap_sem); | |
98 | if (!ret) | |
99 | ret = -EFAULT; | |
100 | if (ret > 0) | |
101 | vec->nr_frames = ret; | |
102 | return ret; | |
103 | } | |
104 | EXPORT_SYMBOL(get_vaddr_frames); | |
105 | ||
106 | /** | |
107 | * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired | |
108 | * them | |
109 | * @vec: frame vector to put | |
110 | * | |
111 | * Drop references to pages if get_vaddr_frames() acquired them. We also | |
112 | * invalidate the frame vector so that it is prepared for the next call into | |
113 | * get_vaddr_frames(). | |
114 | */ | |
115 | void put_vaddr_frames(struct frame_vector *vec) | |
116 | { | |
117 | int i; | |
118 | struct page **pages; | |
119 | ||
120 | if (!vec->got_ref) | |
121 | goto out; | |
122 | pages = frame_vector_pages(vec); | |
123 | /* | |
124 | * frame_vector_pages() might needed to do a conversion when | |
125 | * get_vaddr_frames() got pages but vec was later converted to pfns. | |
126 | * But it shouldn't really fail to convert pfns back... | |
127 | */ | |
128 | if (WARN_ON(IS_ERR(pages))) | |
129 | goto out; | |
130 | for (i = 0; i < vec->nr_frames; i++) | |
131 | put_page(pages[i]); | |
132 | vec->got_ref = false; | |
133 | out: | |
134 | vec->nr_frames = 0; | |
135 | } | |
136 | EXPORT_SYMBOL(put_vaddr_frames); | |
137 | ||
138 | /** | |
139 | * frame_vector_to_pages - convert frame vector to contain page pointers | |
140 | * @vec: frame vector to convert | |
141 | * | |
142 | * Convert @vec to contain array of page pointers. If the conversion is | |
143 | * successful, return 0. Otherwise return an error. Note that we do not grab | |
144 | * page references for the page structures. | |
145 | */ | |
146 | int frame_vector_to_pages(struct frame_vector *vec) | |
147 | { | |
148 | int i; | |
149 | unsigned long *nums; | |
150 | struct page **pages; | |
151 | ||
152 | if (!vec->is_pfns) | |
153 | return 0; | |
154 | nums = frame_vector_pfns(vec); | |
155 | for (i = 0; i < vec->nr_frames; i++) | |
156 | if (!pfn_valid(nums[i])) | |
157 | return -EINVAL; | |
158 | pages = (struct page **)nums; | |
159 | for (i = 0; i < vec->nr_frames; i++) | |
160 | pages[i] = pfn_to_page(nums[i]); | |
161 | vec->is_pfns = false; | |
162 | return 0; | |
163 | } | |
164 | EXPORT_SYMBOL(frame_vector_to_pages); | |
165 | ||
166 | /** | |
167 | * frame_vector_to_pfns - convert frame vector to contain pfns | |
168 | * @vec: frame vector to convert | |
169 | * | |
170 | * Convert @vec to contain array of pfns. | |
171 | */ | |
172 | void frame_vector_to_pfns(struct frame_vector *vec) | |
173 | { | |
174 | int i; | |
175 | unsigned long *nums; | |
176 | struct page **pages; | |
177 | ||
178 | if (vec->is_pfns) | |
179 | return; | |
180 | pages = (struct page **)(vec->ptrs); | |
181 | nums = (unsigned long *)pages; | |
182 | for (i = 0; i < vec->nr_frames; i++) | |
183 | nums[i] = page_to_pfn(pages[i]); | |
184 | vec->is_pfns = true; | |
185 | } | |
186 | EXPORT_SYMBOL(frame_vector_to_pfns); | |
187 | ||
188 | /** | |
189 | * frame_vector_create() - allocate & initialize structure for pinned pfns | |
190 | * @nr_frames: number of pfns slots we should reserve | |
191 | * | |
192 | * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns | |
193 | * pfns. | |
194 | */ | |
195 | struct frame_vector *frame_vector_create(unsigned int nr_frames) | |
196 | { | |
197 | struct frame_vector *vec; | |
198 | int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames; | |
199 | ||
200 | if (WARN_ON_ONCE(nr_frames == 0)) | |
201 | return NULL; | |
202 | /* | |
203 | * This is absurdly high. It's here just to avoid strange effects when | |
204 | * arithmetics overflows. | |
205 | */ | |
206 | if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2)) | |
207 | return NULL; | |
208 | /* | |
209 | * Avoid higher order allocations, use vmalloc instead. It should | |
210 | * be rare anyway. | |
211 | */ | |
212 | if (size <= PAGE_SIZE) | |
213 | vec = kmalloc(size, GFP_KERNEL); | |
214 | else | |
215 | vec = vmalloc(size); | |
216 | if (!vec) | |
217 | return NULL; | |
218 | vec->nr_allocated = nr_frames; | |
219 | vec->nr_frames = 0; | |
220 | return vec; | |
221 | } | |
222 | EXPORT_SYMBOL(frame_vector_create); | |
223 | ||
224 | /** | |
225 | * frame_vector_destroy() - free memory allocated to carry frame vector | |
226 | * @vec: Frame vector to free | |
227 | * | |
228 | * Free structure allocated by frame_vector_create() to carry frames. | |
229 | */ | |
230 | void frame_vector_destroy(struct frame_vector *vec) | |
231 | { | |
232 | /* Make sure put_vaddr_frames() got called properly... */ | |
233 | VM_BUG_ON(vec->nr_frames > 0); | |
234 | kvfree(vec); | |
235 | } | |
236 | EXPORT_SYMBOL(frame_vector_destroy); |