Commit | Line | Data |
---|---|---|
2235acb2 JC |
1 | /* The industrial I/O simple minimally locked ring buffer. |
2 | * | |
3 | * Copyright (c) 2008 Jonathan Cameron | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms of the GNU General Public License version 2 as published by | |
7 | * the Free Software Foundation. | |
8 | */ | |
9 | ||
5a0e3ad6 | 10 | #include <linux/slab.h> |
2235acb2 | 11 | #include <linux/kernel.h> |
2235acb2 JC |
12 | #include <linux/module.h> |
13 | #include <linux/device.h> | |
14 | #include <linux/workqueue.h> | |
15 | #include "ring_sw.h" | |
16 | ||
6f2dfb31 JC |
17 | static inline int __iio_allocate_sw_ring_buffer(struct iio_sw_ring_buffer *ring, |
18 | int bytes_per_datum, int length) | |
2235acb2 JC |
19 | { |
20 | if ((length == 0) || (bytes_per_datum == 0)) | |
21 | return -EINVAL; | |
6f2dfb31 | 22 | __iio_update_ring_buffer(&ring->buf, bytes_per_datum, length); |
2235acb2 | 23 | ring->data = kmalloc(length*ring->buf.bpd, GFP_KERNEL); |
19ca92e0 GKH |
24 | ring->read_p = NULL; |
25 | ring->write_p = NULL; | |
26 | ring->last_written_p = NULL; | |
27 | ring->half_p = NULL; | |
2235acb2 JC |
28 | return ring->data ? 0 : -ENOMEM; |
29 | } | |
30 | ||
6f2dfb31 JC |
31 | static inline void __iio_init_sw_ring_buffer(struct iio_sw_ring_buffer *ring) |
32 | { | |
33 | spin_lock_init(&ring->use_lock); | |
34 | } | |
35 | ||
2235acb2 JC |
36 | static inline void __iio_free_sw_ring_buffer(struct iio_sw_ring_buffer *ring) |
37 | { | |
38 | kfree(ring->data); | |
39 | } | |
40 | ||
41 | void iio_mark_sw_rb_in_use(struct iio_ring_buffer *r) | |
42 | { | |
43 | struct iio_sw_ring_buffer *ring = iio_to_sw_ring(r); | |
44 | spin_lock(&ring->use_lock); | |
45 | ring->use_count++; | |
46 | spin_unlock(&ring->use_lock); | |
47 | } | |
48 | EXPORT_SYMBOL(iio_mark_sw_rb_in_use); | |
49 | ||
50 | void iio_unmark_sw_rb_in_use(struct iio_ring_buffer *r) | |
51 | { | |
52 | struct iio_sw_ring_buffer *ring = iio_to_sw_ring(r); | |
53 | spin_lock(&ring->use_lock); | |
54 | ring->use_count--; | |
55 | spin_unlock(&ring->use_lock); | |
56 | } | |
57 | EXPORT_SYMBOL(iio_unmark_sw_rb_in_use); | |
58 | ||
59 | ||
60 | /* Ring buffer related functionality */ | |
61 | /* Store to ring is typically called in the bh of a data ready interrupt handler | |
62 | * in the device driver */ | |
63 | /* Lock always held if their is a chance this may be called */ | |
64 | /* Only one of these per ring may run concurrently - enforced by drivers */ | |
19ca92e0 GKH |
65 | static int iio_store_to_sw_ring(struct iio_sw_ring_buffer *ring, |
66 | unsigned char *data, s64 timestamp) | |
2235acb2 JC |
67 | { |
68 | int ret = 0; | |
69 | int code; | |
70 | unsigned char *temp_ptr, *change_test_ptr; | |
71 | ||
72 | /* initial store */ | |
19ca92e0 | 73 | if (unlikely(ring->write_p == NULL)) { |
2235acb2 JC |
74 | ring->write_p = ring->data; |
75 | /* Doesn't actually matter if this is out of the set | |
76 | * as long as the read pointer is valid before this | |
77 | * passes it - guaranteed as set later in this function. | |
78 | */ | |
79 | ring->half_p = ring->data - ring->buf.length*ring->buf.bpd/2; | |
80 | } | |
81 | /* Copy data to where ever the current write pointer says */ | |
82 | memcpy(ring->write_p, data, ring->buf.bpd); | |
83 | barrier(); | |
84 | /* Update the pointer used to get most recent value. | |
85 | * Always valid as either points to latest or second latest value. | |
86 | * Before this runs it is null and read attempts fail with -EAGAIN. | |
87 | */ | |
88 | ring->last_written_p = ring->write_p; | |
89 | barrier(); | |
90 | /* temp_ptr used to ensure we never have an invalid pointer | |
91 | * it may be slightly lagging, but never invalid | |
92 | */ | |
93 | temp_ptr = ring->write_p + ring->buf.bpd; | |
94 | /* End of ring, back to the beginning */ | |
95 | if (temp_ptr == ring->data + ring->buf.length*ring->buf.bpd) | |
96 | temp_ptr = ring->data; | |
97 | /* Update the write pointer | |
98 | * always valid as long as this is the only function able to write. | |
99 | * Care needed with smp systems to ensure more than one ring fill | |
100 | * is never scheduled. | |
101 | */ | |
102 | ring->write_p = temp_ptr; | |
103 | ||
19ca92e0 | 104 | if (ring->read_p == NULL) |
2235acb2 JC |
105 | ring->read_p = ring->data; |
106 | /* Buffer full - move the read pointer and create / escalate | |
107 | * ring event */ | |
108 | /* Tricky case - if the read pointer moves before we adjust it. | |
109 | * Handle by not pushing if it has moved - may result in occasional | |
110 | * unnecessary buffer full events when it wasn't quite true. | |
111 | */ | |
112 | else if (ring->write_p == ring->read_p) { | |
113 | change_test_ptr = ring->read_p; | |
114 | temp_ptr = change_test_ptr + ring->buf.bpd; | |
115 | if (temp_ptr | |
116 | == ring->data + ring->buf.length*ring->buf.bpd) { | |
117 | temp_ptr = ring->data; | |
118 | } | |
119 | /* We are moving pointer on one because the ring is full. Any | |
120 | * change to the read pointer will be this or greater. | |
121 | */ | |
122 | if (change_test_ptr == ring->read_p) | |
123 | ring->read_p = temp_ptr; | |
124 | ||
125 | spin_lock(&ring->buf.shared_ev_pointer.lock); | |
126 | ||
127 | ret = iio_push_or_escallate_ring_event(&ring->buf, | |
c849d253 | 128 | IIO_EVENT_CODE_RING_100_FULL, timestamp); |
2235acb2 JC |
129 | spin_unlock(&ring->buf.shared_ev_pointer.lock); |
130 | if (ret) | |
131 | goto error_ret; | |
132 | } | |
133 | /* investigate if our event barrier has been passed */ | |
134 | /* There are definite 'issues' with this and chances of | |
135 | * simultaneous read */ | |
136 | /* Also need to use loop count to ensure this only happens once */ | |
137 | ring->half_p += ring->buf.bpd; | |
138 | if (ring->half_p == ring->data + ring->buf.length*ring->buf.bpd) | |
139 | ring->half_p = ring->data; | |
140 | if (ring->half_p == ring->read_p) { | |
141 | spin_lock(&ring->buf.shared_ev_pointer.lock); | |
142 | code = IIO_EVENT_CODE_RING_50_FULL; | |
143 | ret = __iio_push_event(&ring->buf.ev_int, | |
144 | code, | |
145 | timestamp, | |
146 | &ring->buf.shared_ev_pointer); | |
147 | spin_unlock(&ring->buf.shared_ev_pointer.lock); | |
148 | } | |
149 | error_ret: | |
150 | return ret; | |
151 | } | |
152 | ||
153 | int iio_rip_sw_rb(struct iio_ring_buffer *r, | |
154 | size_t count, u8 **data, int *dead_offset) | |
155 | { | |
156 | struct iio_sw_ring_buffer *ring = iio_to_sw_ring(r); | |
157 | ||
158 | u8 *initial_read_p, *initial_write_p, *current_read_p, *end_read_p; | |
159 | int ret, max_copied; | |
160 | int bytes_to_rip; | |
161 | ||
162 | /* A userspace program has probably made an error if it tries to | |
163 | * read something that is not a whole number of bpds. | |
164 | * Return an error. | |
165 | */ | |
166 | if (count % ring->buf.bpd) { | |
167 | ret = -EINVAL; | |
168 | printk(KERN_INFO "Ring buffer read request not whole number of" | |
169 | "samples: Request bytes %zd, Current bpd %d\n", | |
170 | count, ring->buf.bpd); | |
171 | goto error_ret; | |
172 | } | |
173 | /* Limit size to whole of ring buffer */ | |
174 | bytes_to_rip = min((size_t)(ring->buf.bpd*ring->buf.length), count); | |
175 | ||
176 | *data = kmalloc(bytes_to_rip, GFP_KERNEL); | |
177 | if (*data == NULL) { | |
178 | ret = -ENOMEM; | |
179 | goto error_ret; | |
180 | } | |
181 | ||
182 | /* build local copy */ | |
183 | initial_read_p = ring->read_p; | |
19ca92e0 | 184 | if (unlikely(initial_read_p == NULL)) { /* No data here as yet */ |
2235acb2 JC |
185 | ret = 0; |
186 | goto error_free_data_cpy; | |
187 | } | |
188 | ||
189 | initial_write_p = ring->write_p; | |
190 | ||
191 | /* Need a consistent pair */ | |
192 | while ((initial_read_p != ring->read_p) | |
193 | || (initial_write_p != ring->write_p)) { | |
194 | initial_read_p = ring->read_p; | |
195 | initial_write_p = ring->write_p; | |
196 | } | |
197 | if (initial_write_p == initial_read_p) { | |
198 | /* No new data available.*/ | |
199 | ret = 0; | |
200 | goto error_free_data_cpy; | |
201 | } | |
202 | ||
203 | if (initial_write_p >= initial_read_p + bytes_to_rip) { | |
204 | /* write_p is greater than necessary, all is easy */ | |
205 | max_copied = bytes_to_rip; | |
206 | memcpy(*data, initial_read_p, max_copied); | |
207 | end_read_p = initial_read_p + max_copied; | |
208 | } else if (initial_write_p > initial_read_p) { | |
209 | /*not enough data to cpy */ | |
210 | max_copied = initial_write_p - initial_read_p; | |
211 | memcpy(*data, initial_read_p, max_copied); | |
212 | end_read_p = initial_write_p; | |
213 | } else { | |
214 | /* going through 'end' of ring buffer */ | |
215 | max_copied = ring->data | |
216 | + ring->buf.length*ring->buf.bpd - initial_read_p; | |
217 | memcpy(*data, initial_read_p, max_copied); | |
218 | /* possible we are done if we align precisely with end */ | |
219 | if (max_copied == bytes_to_rip) | |
220 | end_read_p = ring->data; | |
221 | else if (initial_write_p | |
222 | > ring->data + bytes_to_rip - max_copied) { | |
223 | /* enough data to finish */ | |
224 | memcpy(*data + max_copied, ring->data, | |
225 | bytes_to_rip - max_copied); | |
226 | max_copied = bytes_to_rip; | |
227 | end_read_p = ring->data + (bytes_to_rip - max_copied); | |
228 | } else { /* not enough data */ | |
229 | memcpy(*data + max_copied, ring->data, | |
230 | initial_write_p - ring->data); | |
231 | max_copied += initial_write_p - ring->data; | |
232 | end_read_p = initial_write_p; | |
233 | } | |
234 | } | |
235 | /* Now to verify which section was cleanly copied - i.e. how far | |
236 | * read pointer has been pushed */ | |
237 | current_read_p = ring->read_p; | |
238 | ||
239 | if (initial_read_p <= current_read_p) | |
240 | *dead_offset = current_read_p - initial_read_p; | |
241 | else | |
242 | *dead_offset = ring->buf.length*ring->buf.bpd | |
243 | - (initial_read_p - current_read_p); | |
244 | ||
245 | /* possible issue if the initial write has been lapped or indeed | |
246 | * the point we were reading to has been passed */ | |
247 | /* No valid data read. | |
248 | * In this case the read pointer is already correct having been | |
249 | * pushed further than we would look. */ | |
250 | if (max_copied - *dead_offset < 0) { | |
251 | ret = 0; | |
252 | goto error_free_data_cpy; | |
253 | } | |
254 | ||
255 | /* setup the next read position */ | |
256 | /* Beware, this may fail due to concurrency fun and games. | |
257 | * Possible that sufficient fill commands have run to push the read | |
258 | * pointer past where we would be after the rip. If this occurs, leave | |
259 | * it be. | |
260 | */ | |
261 | /* Tricky - deal with loops */ | |
262 | ||
263 | while (ring->read_p != end_read_p) | |
264 | ring->read_p = end_read_p; | |
265 | ||
266 | return max_copied - *dead_offset; | |
267 | ||
268 | error_free_data_cpy: | |
269 | kfree(*data); | |
270 | error_ret: | |
271 | return ret; | |
272 | } | |
273 | EXPORT_SYMBOL(iio_rip_sw_rb); | |
274 | ||
275 | int iio_store_to_sw_rb(struct iio_ring_buffer *r, u8 *data, s64 timestamp) | |
276 | { | |
277 | struct iio_sw_ring_buffer *ring = iio_to_sw_ring(r); | |
278 | return iio_store_to_sw_ring(ring, data, timestamp); | |
279 | } | |
280 | EXPORT_SYMBOL(iio_store_to_sw_rb); | |
281 | ||
19ca92e0 GKH |
282 | static int iio_read_last_from_sw_ring(struct iio_sw_ring_buffer *ring, |
283 | unsigned char *data) | |
2235acb2 JC |
284 | { |
285 | unsigned char *last_written_p_copy; | |
286 | ||
287 | iio_mark_sw_rb_in_use(&ring->buf); | |
288 | again: | |
289 | barrier(); | |
290 | last_written_p_copy = ring->last_written_p; | |
291 | barrier(); /*unnessecary? */ | |
292 | /* Check there is anything here */ | |
19ca92e0 | 293 | if (last_written_p_copy == NULL) |
2235acb2 JC |
294 | return -EAGAIN; |
295 | memcpy(data, last_written_p_copy, ring->buf.bpd); | |
296 | ||
8474ddd7 | 297 | if (unlikely(ring->last_written_p != last_written_p_copy)) |
2235acb2 JC |
298 | goto again; |
299 | ||
300 | iio_unmark_sw_rb_in_use(&ring->buf); | |
301 | return 0; | |
302 | } | |
303 | ||
304 | int iio_read_last_from_sw_rb(struct iio_ring_buffer *r, | |
305 | unsigned char *data) | |
306 | { | |
307 | return iio_read_last_from_sw_ring(iio_to_sw_ring(r), data); | |
308 | } | |
309 | EXPORT_SYMBOL(iio_read_last_from_sw_rb); | |
310 | ||
311 | int iio_request_update_sw_rb(struct iio_ring_buffer *r) | |
312 | { | |
313 | int ret = 0; | |
314 | struct iio_sw_ring_buffer *ring = iio_to_sw_ring(r); | |
315 | ||
316 | spin_lock(&ring->use_lock); | |
317 | if (!ring->update_needed) | |
318 | goto error_ret; | |
319 | if (ring->use_count) { | |
320 | ret = -EAGAIN; | |
321 | goto error_ret; | |
322 | } | |
323 | __iio_free_sw_ring_buffer(ring); | |
6f2dfb31 JC |
324 | ret = __iio_allocate_sw_ring_buffer(ring, ring->buf.bpd, |
325 | ring->buf.length); | |
2235acb2 JC |
326 | error_ret: |
327 | spin_unlock(&ring->use_lock); | |
328 | return ret; | |
329 | } | |
330 | EXPORT_SYMBOL(iio_request_update_sw_rb); | |
331 | ||
332 | int iio_get_bpd_sw_rb(struct iio_ring_buffer *r) | |
333 | { | |
334 | struct iio_sw_ring_buffer *ring = iio_to_sw_ring(r); | |
335 | return ring->buf.bpd; | |
336 | } | |
337 | EXPORT_SYMBOL(iio_get_bpd_sw_rb); | |
338 | ||
339 | int iio_set_bpd_sw_rb(struct iio_ring_buffer *r, size_t bpd) | |
340 | { | |
341 | if (r->bpd != bpd) { | |
342 | r->bpd = bpd; | |
343 | if (r->access.mark_param_change) | |
344 | r->access.mark_param_change(r); | |
345 | } | |
346 | return 0; | |
347 | } | |
348 | EXPORT_SYMBOL(iio_set_bpd_sw_rb); | |
349 | ||
350 | int iio_get_length_sw_rb(struct iio_ring_buffer *r) | |
351 | { | |
352 | return r->length; | |
353 | } | |
354 | EXPORT_SYMBOL(iio_get_length_sw_rb); | |
355 | ||
356 | int iio_set_length_sw_rb(struct iio_ring_buffer *r, int length) | |
357 | { | |
358 | if (r->length != length) { | |
359 | r->length = length; | |
360 | if (r->access.mark_param_change) | |
361 | r->access.mark_param_change(r); | |
362 | } | |
363 | return 0; | |
364 | } | |
365 | EXPORT_SYMBOL(iio_set_length_sw_rb); | |
366 | ||
367 | int iio_mark_update_needed_sw_rb(struct iio_ring_buffer *r) | |
368 | { | |
369 | struct iio_sw_ring_buffer *ring = iio_to_sw_ring(r); | |
370 | ring->update_needed = true; | |
371 | return 0; | |
372 | } | |
373 | EXPORT_SYMBOL(iio_mark_update_needed_sw_rb); | |
374 | ||
375 | static void iio_sw_rb_release(struct device *dev) | |
376 | { | |
377 | struct iio_ring_buffer *r = to_iio_ring_buffer(dev); | |
378 | kfree(iio_to_sw_ring(r)); | |
379 | } | |
380 | ||
381 | static IIO_RING_ENABLE_ATTR; | |
382 | static IIO_RING_BPS_ATTR; | |
383 | static IIO_RING_LENGTH_ATTR; | |
384 | ||
385 | /* Standard set of ring buffer attributes */ | |
386 | static struct attribute *iio_ring_attributes[] = { | |
387 | &dev_attr_length.attr, | |
388 | &dev_attr_bps.attr, | |
389 | &dev_attr_ring_enable.attr, | |
390 | NULL, | |
391 | }; | |
392 | ||
393 | static struct attribute_group iio_ring_attribute_group = { | |
394 | .attrs = iio_ring_attributes, | |
395 | }; | |
396 | ||
3860dc82 | 397 | static const struct attribute_group *iio_ring_attribute_groups[] = { |
2235acb2 JC |
398 | &iio_ring_attribute_group, |
399 | NULL | |
400 | }; | |
401 | ||
402 | static struct device_type iio_sw_ring_type = { | |
403 | .release = iio_sw_rb_release, | |
404 | .groups = iio_ring_attribute_groups, | |
405 | }; | |
406 | ||
407 | struct iio_ring_buffer *iio_sw_rb_allocate(struct iio_dev *indio_dev) | |
408 | { | |
409 | struct iio_ring_buffer *buf; | |
410 | struct iio_sw_ring_buffer *ring; | |
411 | ||
412 | ring = kzalloc(sizeof *ring, GFP_KERNEL); | |
413 | if (!ring) | |
19ca92e0 | 414 | return NULL; |
2235acb2 | 415 | buf = &ring->buf; |
2235acb2 | 416 | iio_ring_buffer_init(buf, indio_dev); |
6f2dfb31 | 417 | __iio_init_sw_ring_buffer(ring); |
2235acb2 JC |
418 | buf->dev.type = &iio_sw_ring_type; |
419 | device_initialize(&buf->dev); | |
420 | buf->dev.parent = &indio_dev->dev; | |
5aaaeba8 | 421 | buf->dev.bus = &iio_bus_type; |
2235acb2 JC |
422 | dev_set_drvdata(&buf->dev, (void *)buf); |
423 | ||
424 | return buf; | |
425 | } | |
426 | EXPORT_SYMBOL(iio_sw_rb_allocate); | |
427 | ||
428 | void iio_sw_rb_free(struct iio_ring_buffer *r) | |
429 | { | |
430 | if (r) | |
431 | iio_put_ring_buffer(r); | |
432 | } | |
433 | EXPORT_SYMBOL(iio_sw_rb_free); | |
434 | MODULE_DESCRIPTION("Industrialio I/O software ring buffer"); | |
435 | MODULE_LICENSE("GPL"); |