1 From 206b4c81b6b31d87c758748cdbc6d25e9c721ea1 Mon Sep 17 00:00:00 2001
2 In-Reply-To: <20160106001143.GA1171@kroah.com>
3 References: <20160106001143.GA1171@kroah.com>
4 From: "Steinar H. Gunderson" <sesse@google.com>
5 Date: Thu, 26 Nov 2015 01:19:13 +0100
6 Subject: [PATCH v2] Add support for usbfs zerocopy.
7 To: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
8 Cc: linux-usb@vger.kernel.org,linux-kernel@vger.kernel.org,stern@rowland.harvard.edu
10 Add a new interface for userspace to preallocate memory that can be
11 used with usbfs. This gives two primary benefits:
13 - Zerocopy; data no longer needs to be copied between the userspace
14 and the kernel, but can instead be read directly by the driver from
15 userspace's buffers. This works for all kinds of transfers (even if
16 nonsensical for control and interrupt transfers); isochronous also
17 no longer need to memset() the buffer to zero to avoid leaking kernel data.
19 - Once the buffers are allocated, USB transfers can no longer fail due to
20 memory fragmentation; previously, long-running programs could run into
21 problems finding a large enough contiguous memory chunk, especially on
22 embedded systems or at high rates.
24 Memory is allocated by using mmap() against the usbfs file descriptor,
25 and similarly deallocated by munmap(). Once memory has been allocated,
26 using it as pointers to a bulk or isochronous operation means you will
27 automatically get zerocopy behavior. Note that this also means you cannot
28 modify outgoing data until the transfer is complete. The same holds for
29 data on the same cache lines as incoming data; DMA modifying them at the
30 same time could lead to your changes being overwritten.
32 There's a new capability USBDEVFS_CAP_MMAP that userspace can query to see
33 if the running kernel supports this functionality, if just trying mmap() is
36 Largely based on a patch by Markus Rechberger with some updates. The original
37 patch can be found at:
39 http://sundtek.de/support/devio_mmap_v0.4.diff
41 Signed-off-by: Steinar H. Gunderson <sesse@google.com>
42 Signed-off-by: Markus Rechberger <mrechberger@gmail.com>
43 Acked-by: Alan Stern <stern@rowland.harvard.edu>
45 drivers/usb/core/devio.c | 227 +++++++++++++++++++++++++++++++++-----
46 include/uapi/linux/usbdevice_fs.h | 1 +
47 2 files changed, 203 insertions(+), 25 deletions(-)
49 diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
50 index 38ae877c..0238c78 100644
51 --- a/drivers/usb/core/devio.c
52 +++ b/drivers/usb/core/devio.c
54 #include <linux/user_namespace.h>
55 #include <linux/scatterlist.h>
56 #include <linux/uaccess.h>
57 +#include <linux/dma-mapping.h>
58 #include <asm/byteorder.h>
59 #include <linux/moduleparam.h>
61 @@ -69,6 +70,7 @@ struct usb_dev_state {
62 spinlock_t lock; /* protects the async urb lists */
63 struct list_head async_pending;
64 struct list_head async_completed;
65 + struct list_head memory_list;
66 wait_queue_head_t wait; /* wake up if a request completed */
67 unsigned int discsignr;
69 @@ -79,6 +81,17 @@ struct usb_dev_state {
70 u32 disabled_bulk_eps;
74 + struct list_head memlist;
79 + dma_addr_t dma_handle;
80 + unsigned long vm_start;
81 + struct usb_dev_state *ps;
85 struct list_head asynclist;
86 struct usb_dev_state *ps;
87 @@ -89,6 +102,7 @@ struct async {
88 void __user *userbuffer;
91 + struct usb_memory *usbm;
92 unsigned int mem_usage;
95 @@ -157,6 +171,111 @@ static int connected(struct usb_dev_state *ps)
96 ps->dev->state != USB_STATE_NOTATTACHED);
99 +static void dec_usb_memory_use_count(struct usb_memory *usbm, int *count)
101 + struct usb_dev_state *ps = usbm->ps;
102 + unsigned long flags;
104 + spin_lock_irqsave(&ps->lock, flags);
106 + if (usbm->urb_use_count == 0 && usbm->vma_use_count == 0) {
107 + list_del(&usbm->memlist);
108 + spin_unlock_irqrestore(&ps->lock, flags);
110 + usb_free_coherent(ps->dev, usbm->size, usbm->mem,
112 + usbfs_decrease_memory_usage(
113 + usbm->size + sizeof(struct usb_memory));
116 + spin_unlock_irqrestore(&ps->lock, flags);
120 +static void usbdev_vm_open(struct vm_area_struct *vma)
122 + struct usb_memory *usbm = vma->vm_private_data;
123 + unsigned long flags;
125 + spin_lock_irqsave(&usbm->ps->lock, flags);
126 + ++usbm->vma_use_count;
127 + spin_unlock_irqrestore(&usbm->ps->lock, flags);
130 +static void usbdev_vm_close(struct vm_area_struct *vma)
132 + struct usb_memory *usbm = vma->vm_private_data;
134 + dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
137 +struct vm_operations_struct usbdev_vm_ops = {
138 + .open = usbdev_vm_open,
139 + .close = usbdev_vm_close
142 +static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
144 + struct usb_memory *usbm = NULL;
145 + struct usb_dev_state *ps = file->private_data;
146 + size_t size = vma->vm_end - vma->vm_start;
148 + unsigned long flags;
149 + dma_addr_t dma_handle;
152 + ret = usbfs_increase_memory_usage(size + sizeof(struct usb_memory));
156 + usbm = kzalloc(sizeof(struct usb_memory), GFP_KERNEL);
159 + goto error_decrease_mem;
162 + mem = usb_alloc_coherent(ps->dev, size, GFP_USER, &dma_handle);
165 + goto error_free_usbm;
168 + memset(mem, 0, size);
171 + usbm->dma_handle = dma_handle;
174 + usbm->vm_start = vma->vm_start;
175 + usbm->vma_use_count = 1;
176 + INIT_LIST_HEAD(&usbm->memlist);
178 + if (remap_pfn_range(vma, vma->vm_start,
179 + virt_to_phys(usbm->mem) >> PAGE_SHIFT,
180 + size, vma->vm_page_prot) < 0) {
181 + dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
185 + vma->vm_flags |= VM_IO;
186 + vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
187 + vma->vm_ops = &usbdev_vm_ops;
188 + vma->vm_private_data = usbm;
190 + spin_lock_irqsave(&ps->lock, flags);
191 + list_add_tail(&usbm->memlist, &ps->memory_list);
192 + spin_unlock_irqrestore(&ps->lock, flags);
199 + usbfs_decrease_memory_usage(size + sizeof(struct usb_memory));
204 static loff_t usbdev_lseek(struct file *file, loff_t offset, int orig)
207 @@ -297,8 +416,13 @@ static void free_async(struct async *as)
208 if (sg_page(&as->urb->sg[i]))
209 kfree(sg_virt(&as->urb->sg[i]));
213 - kfree(as->urb->transfer_buffer);
214 + if (as->usbm == NULL)
215 + kfree(as->urb->transfer_buffer);
217 + dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
219 kfree(as->urb->setup_packet);
220 usb_free_urb(as->urb);
221 usbfs_decrease_memory_usage(as->mem_usage);
222 @@ -910,6 +1034,7 @@ static int usbdev_open(struct inode *inode, struct file *file)
223 INIT_LIST_HEAD(&ps->list);
224 INIT_LIST_HEAD(&ps->async_pending);
225 INIT_LIST_HEAD(&ps->async_completed);
226 + INIT_LIST_HEAD(&ps->memory_list);
227 init_waitqueue_head(&ps->wait);
229 ps->disc_pid = get_pid(task_pid(current));
230 @@ -962,6 +1087,7 @@ static int usbdev_release(struct inode *inode, struct file *file)
232 as = async_getcompleted(ps);
238 @@ -1283,6 +1409,31 @@ static int proc_setconfig(struct usb_dev_state *ps, void __user *arg)
242 +static struct usb_memory *
243 +find_memory_area(struct usb_dev_state *ps, const struct usbdevfs_urb *uurb)
245 + struct usb_memory *usbm = NULL, *iter;
246 + unsigned long flags;
247 + unsigned long uurb_start = (unsigned long)uurb->buffer;
249 + spin_lock_irqsave(&ps->lock, flags);
250 + list_for_each_entry(iter, &ps->memory_list, memlist) {
251 + if (uurb_start >= iter->vm_start &&
252 + uurb_start < iter->vm_start + iter->size) {
253 + if (uurb->buffer_length > iter->vm_start + iter->size -
255 + usbm = ERR_PTR(-EINVAL);
258 + usbm->urb_use_count++;
263 + spin_unlock_irqrestore(&ps->lock, flags);
267 static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb,
268 struct usbdevfs_iso_packet_desc __user *iso_frame_desc,
270 @@ -1439,6 +1590,19 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
274 + as->usbm = find_memory_area(ps, uurb);
275 + if (IS_ERR(as->usbm)) {
276 + ret = PTR_ERR(as->usbm);
281 + /* do not use SG buffers when memory mapped segments
287 u += sizeof(struct async) + sizeof(struct urb) + uurb->buffer_length +
288 num_sgs * sizeof(struct scatterlist);
289 ret = usbfs_increase_memory_usage(u);
290 @@ -1476,29 +1640,35 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
293 } else if (uurb->buffer_length > 0) {
294 - as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
296 - if (!as->urb->transfer_buffer) {
301 + unsigned long uurb_start = (unsigned long)uurb->buffer;
304 - if (copy_from_user(as->urb->transfer_buffer,
306 - uurb->buffer_length)) {
308 + as->urb->transfer_buffer = as->usbm->mem +
309 + (uurb_start - as->usbm->vm_start);
311 + as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
313 + if (!as->urb->transfer_buffer) {
317 - } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
319 - * Isochronous input data may end up being
320 - * discontiguous if some of the packets are short.
321 - * Clear the buffer so that the gaps don't leak
322 - * kernel data to userspace.
324 - memset(as->urb->transfer_buffer, 0,
325 - uurb->buffer_length);
327 + if (copy_from_user(as->urb->transfer_buffer,
329 + uurb->buffer_length)) {
333 + } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
335 + * Isochronous input data may end up being
336 + * discontiguous if some of the packets are
337 + * short. Clear the buffer so that the gaps
338 + * don't leak kernel data to userspace.
340 + memset(as->urb->transfer_buffer, 0,
341 + uurb->buffer_length);
345 as->urb->dev = ps->dev;
346 @@ -1545,10 +1715,14 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
350 - if (is_in && uurb->buffer_length > 0)
352 + unsigned long uurb_start = (unsigned long)uurb->buffer;
354 + as->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
355 + as->urb->transfer_dma = as->usbm->dma_handle +
356 + (uurb_start - as->usbm->vm_start);
357 + } else if (is_in && uurb->buffer_length > 0)
358 as->userbuffer = uurb->buffer;
360 - as->userbuffer = NULL;
361 as->signr = uurb->signr;
363 as->pid = get_pid(task_pid(current));
364 @@ -1604,6 +1778,8 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
368 + if (as && as->usbm)
369 + dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
373 @@ -2047,7 +2223,7 @@ static int proc_get_capabilities(struct usb_dev_state *ps, void __user *arg)
376 caps = USBDEVFS_CAP_ZERO_PACKET | USBDEVFS_CAP_NO_PACKET_SIZE_LIM |
377 - USBDEVFS_CAP_REAP_AFTER_DISCONNECT;
378 + USBDEVFS_CAP_REAP_AFTER_DISCONNECT | USBDEVFS_CAP_MMAP;
379 if (!ps->dev->bus->no_stop_on_short)
380 caps |= USBDEVFS_CAP_BULK_CONTINUATION;
381 if (ps->dev->bus->sg_tablesize)
382 @@ -2373,6 +2549,7 @@ const struct file_operations usbdev_file_operations = {
384 .compat_ioctl = usbdev_compat_ioctl,
386 + .mmap = usbdev_mmap,
388 .release = usbdev_release,
390 diff --git a/include/uapi/linux/usbdevice_fs.h b/include/uapi/linux/usbdevice_fs.h
391 index 019ba1e..ecbd176 100644
392 --- a/include/uapi/linux/usbdevice_fs.h
393 +++ b/include/uapi/linux/usbdevice_fs.h
394 @@ -134,6 +134,7 @@ struct usbdevfs_hub_portinfo {
395 #define USBDEVFS_CAP_NO_PACKET_SIZE_LIM 0x04
396 #define USBDEVFS_CAP_BULK_SCATTER_GATHER 0x08
397 #define USBDEVFS_CAP_REAP_AFTER_DISCONNECT 0x10
398 +#define USBDEVFS_CAP_MMAP 0x20
400 /* USBDEVFS_DISCONNECT_CLAIM flags & struct */