git.sesse.net Git - nageru/blob - patches/linux-add-support-for-persistent-device-memory.diff

   1 From 206b4c81b6b31d87c758748cdbc6d25e9c721ea1 Mon Sep 17 00:00:00 2001
   2 In-Reply-To: <20160106001143.GA1171@kroah.com>
   3 References: <20160106001143.GA1171@kroah.com>
   4 From: "Steinar H. Gunderson" <sesse@google.com>
   5 Date: Thu, 26 Nov 2015 01:19:13 +0100
   6 Subject: [PATCH v2] Add support for usbfs zerocopy.
   7 To: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
   8 Cc: linux-usb@vger.kernel.org,linux-kernel@vger.kernel.org,stern@rowland.harvard.edu
   9
  10 Add a new interface for userspace to preallocate memory that can be
  11 used with usbfs. This gives two primary benefits:
  12
  13  - Zerocopy; data no longer needs to be copied between the userspace
  14    and the kernel, but can instead be read directly by the driver from
  15    userspace's buffers. This works for all kinds of transfers (even if
  16    nonsensical for control and interrupt transfers); isochronous also
  17    no longer need to memset() the buffer to zero to avoid leaking kernel data.
  18
  19  - Once the buffers are allocated, USB transfers can no longer fail due to
  20    memory fragmentation; previously, long-running programs could run into
  21    problems finding a large enough contiguous memory chunk, especially on
  22    embedded systems or at high rates.
  23
  24 Memory is allocated by using mmap() against the usbfs file descriptor,
  25 and similarly deallocated by munmap(). Once memory has been allocated,
  26 using it as pointers to a bulk or isochronous operation means you will
  27 automatically get zerocopy behavior. Note that this also means you cannot
  28 modify outgoing data until the transfer is complete. The same holds for
  29 data on the same cache lines as incoming data; DMA modifying them at the
  30 same time could lead to your changes being overwritten.
  31
  32 There's a new capability USBDEVFS_CAP_MMAP that userspace can query to see
  33 if the running kernel supports this functionality, if just trying mmap() is
  34 not acceptable.
  35
  36 Largely based on a patch by Markus Rechberger with some updates. The original
  37 patch can be found at:
  38
  39   http://sundtek.de/support/devio_mmap_v0.4.diff
  40
  41 Signed-off-by: Steinar H. Gunderson <sesse@google.com>
  42 Signed-off-by: Markus Rechberger <mrechberger@gmail.com>
  43 Acked-by: Alan Stern <stern@rowland.harvard.edu>
  44 ---
  45  drivers/usb/core/devio.c          | 227 +++++++++++++++++++++++++++++++++-----
  46  include/uapi/linux/usbdevice_fs.h |   1 +
  47  2 files changed, 203 insertions(+), 25 deletions(-)
  48
  49 diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
  50 index 38ae877c..0238c78 100644
  51 --- a/drivers/usb/core/devio.c
  52 +++ b/drivers/usb/core/devio.c
  53 @@ -50,6 +50,7 @@
  54  #include <linux/user_namespace.h>
  55  #include <linux/scatterlist.h>
  56  #include <linux/uaccess.h>
  57 +#include <linux/dma-mapping.h>
  58  #include <asm/byteorder.h>
  59  #include <linux/moduleparam.h>
  60
  61 @@ -69,6 +70,7 @@ struct usb_dev_state {
  62         spinlock_t lock;            /* protects the async urb lists */
  63         struct list_head async_pending;
  64         struct list_head async_completed;
  65 +       struct list_head memory_list;
  66         wait_queue_head_t wait;     /* wake up if a request completed */
  67         unsigned int discsignr;
  68         struct pid *disc_pid;
  69 @@ -79,6 +81,17 @@ struct usb_dev_state {
  70         u32 disabled_bulk_eps;
  71  };
  72
  73 +struct usb_memory {
  74 +       struct list_head memlist;
  75 +       int vma_use_count;
  76 +       int urb_use_count;
  77 +       u32 size;
  78 +       void *mem;
  79 +       dma_addr_t dma_handle;
  80 +       unsigned long vm_start;
  81 +       struct usb_dev_state *ps;
  82 +};
  83 +
  84  struct async {
  85         struct list_head asynclist;
  86         struct usb_dev_state *ps;
  87 @@ -89,6 +102,7 @@ struct async {
  88         void __user *userbuffer;
  89         void __user *userurb;
  90         struct urb *urb;
  91 +       struct usb_memory *usbm;
  92         unsigned int mem_usage;
  93         int status;
  94         u32 secid;
  95 @@ -157,6 +171,111 @@ static int connected(struct usb_dev_state *ps)
  96                         ps->dev->state != USB_STATE_NOTATTACHED);
  97  }
  98
  99 +static void dec_usb_memory_use_count(struct usb_memory *usbm, int *count)
 100 +{
 101 +       struct usb_dev_state *ps = usbm->ps;
 102 +       unsigned long flags;
 103 +
 104 +       spin_lock_irqsave(&ps->lock, flags);
 105 +       --*count;
 106 +       if (usbm->urb_use_count == 0 && usbm->vma_use_count == 0) {
 107 +               list_del(&usbm->memlist);
 108 +               spin_unlock_irqrestore(&ps->lock, flags);
 109 +
 110 +               usb_free_coherent(ps->dev, usbm->size, usbm->mem,
 111 +                               usbm->dma_handle);
 112 +               usbfs_decrease_memory_usage(
 113 +                       usbm->size + sizeof(struct usb_memory));
 114 +               kfree(usbm);
 115 +       } else {
 116 +               spin_unlock_irqrestore(&ps->lock, flags);
 117 +       }
 118 +}
 119 +
 120 +static void usbdev_vm_open(struct vm_area_struct *vma)
 121 +{
 122 +       struct usb_memory *usbm = vma->vm_private_data;
 123 +       unsigned long flags;
 124 +
 125 +       spin_lock_irqsave(&usbm->ps->lock, flags);
 126 +       ++usbm->vma_use_count;
 127 +       spin_unlock_irqrestore(&usbm->ps->lock, flags);
 128 +}
 129 +
 130 +static void usbdev_vm_close(struct vm_area_struct *vma)
 131 +{
 132 +       struct usb_memory *usbm = vma->vm_private_data;
 133 +
 134 +       dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
 135 +}
 136 +
 137 +struct vm_operations_struct usbdev_vm_ops = {
 138 +       .open = usbdev_vm_open,
 139 +       .close = usbdev_vm_close
 140 +};
 141 +
 142 +static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
 143 +{
 144 +       struct usb_memory *usbm = NULL;
 145 +       struct usb_dev_state *ps = file->private_data;
 146 +       size_t size = vma->vm_end - vma->vm_start;
 147 +       void *mem;
 148 +       unsigned long flags;
 149 +       dma_addr_t dma_handle;
 150 +       int ret;
 151 +
 152 +       ret = usbfs_increase_memory_usage(size + sizeof(struct usb_memory));
 153 +       if (ret)
 154 +               goto error;
 155 +
 156 +       usbm = kzalloc(sizeof(struct usb_memory), GFP_KERNEL);
 157 +       if (!usbm) {
 158 +               ret = -ENOMEM;
 159 +               goto error_decrease_mem;
 160 +       }
 161 +
 162 +       mem = usb_alloc_coherent(ps->dev, size, GFP_USER, &dma_handle);
 163 +       if (!mem) {
 164 +               ret = -ENOMEM;
 165 +               goto error_free_usbm;
 166 +       }
 167 +
 168 +       memset(mem, 0, size);
 169 +
 170 +       usbm->mem = mem;
 171 +       usbm->dma_handle = dma_handle;
 172 +       usbm->size = size;
 173 +       usbm->ps = ps;
 174 +       usbm->vm_start = vma->vm_start;
 175 +       usbm->vma_use_count = 1;
 176 +       INIT_LIST_HEAD(&usbm->memlist);
 177 +
 178 +       if (remap_pfn_range(vma, vma->vm_start,
 179 +                       virt_to_phys(usbm->mem) >> PAGE_SHIFT,
 180 +                       size, vma->vm_page_prot) < 0) {
 181 +               dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
 182 +               return -EAGAIN;
 183 +       }
 184 +
 185 +       vma->vm_flags |= VM_IO;
 186 +       vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
 187 +       vma->vm_ops = &usbdev_vm_ops;
 188 +       vma->vm_private_data = usbm;
 189 +
 190 +       spin_lock_irqsave(&ps->lock, flags);
 191 +       list_add_tail(&usbm->memlist, &ps->memory_list);
 192 +       spin_unlock_irqrestore(&ps->lock, flags);
 193 +
 194 +       return 0;
 195 +
 196 +error_free_usbm:
 197 +       kfree(usbm);
 198 +error_decrease_mem:
 199 +       usbfs_decrease_memory_usage(size + sizeof(struct usb_memory));
 200 +error:
 201 +       return ret;
 202 +}
 203 +
 204  static loff_t usbdev_lseek(struct file *file, loff_t offset, int orig)
 205  {
 206         loff_t ret;
 207 @@ -297,8 +416,13 @@ static void free_async(struct async *as)
 208                 if (sg_page(&as->urb->sg[i]))
 209                         kfree(sg_virt(&as->urb->sg[i]));
 210         }
 211 +
 212         kfree(as->urb->sg);
 213 -       kfree(as->urb->transfer_buffer);
 214 +       if (as->usbm == NULL)
 215 +               kfree(as->urb->transfer_buffer);
 216 +       else
 217 +               dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
 218 +
 219         kfree(as->urb->setup_packet);
 220         usb_free_urb(as->urb);
 221         usbfs_decrease_memory_usage(as->mem_usage);
 222 @@ -910,6 +1034,7 @@ static int usbdev_open(struct inode *inode, struct file *file)
 223         INIT_LIST_HEAD(&ps->list);
 224         INIT_LIST_HEAD(&ps->async_pending);
 225         INIT_LIST_HEAD(&ps->async_completed);
 226 +       INIT_LIST_HEAD(&ps->memory_list);
 227         init_waitqueue_head(&ps->wait);
 228         ps->discsignr = 0;
 229         ps->disc_pid = get_pid(task_pid(current));
 230 @@ -962,6 +1087,7 @@ static int usbdev_release(struct inode *inode, struct file *file)
 231                 free_async(as);
 232                 as = async_getcompleted(ps);
 233         }
 234 +
 235         kfree(ps);
 236         return 0;
 237  }
 238 @@ -1283,6 +1409,31 @@ static int proc_setconfig(struct usb_dev_state *ps, void __user *arg)
 239         return status;
 240  }
 241
 242 +static struct usb_memory *
 243 +find_memory_area(struct usb_dev_state *ps, const struct usbdevfs_urb *uurb)
 244 +{
 245 +       struct usb_memory *usbm = NULL, *iter;
 246 +       unsigned long flags;
 247 +       unsigned long uurb_start = (unsigned long)uurb->buffer;
 248 +
 249 +       spin_lock_irqsave(&ps->lock, flags);
 250 +       list_for_each_entry(iter, &ps->memory_list, memlist) {
 251 +               if (uurb_start >= iter->vm_start &&
 252 +                               uurb_start < iter->vm_start + iter->size) {
 253 +                       if (uurb->buffer_length > iter->vm_start + iter->size -
 254 +                                       uurb_start) {
 255 +                               usbm = ERR_PTR(-EINVAL);
 256 +                       } else {
 257 +                               usbm = iter;
 258 +                               usbm->urb_use_count++;
 259 +                       }
 260 +                       break;
 261 +               }
 262 +       }
 263 +       spin_unlock_irqrestore(&ps->lock, flags);
 264 +       return usbm;
 265 +}
 266 +
 267  static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb,
 268                         struct usbdevfs_iso_packet_desc __user *iso_frame_desc,
 269                         void __user *arg)
 270 @@ -1439,6 +1590,19 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
 271                 goto error;
 272         }
 273
 274 +       as->usbm = find_memory_area(ps, uurb);
 275 +       if (IS_ERR(as->usbm)) {
 276 +               ret = PTR_ERR(as->usbm);
 277 +               as->usbm = NULL;
 278 +               goto error;
 279 +       }
 280 +
 281 +       /* do not use SG buffers when memory mapped segments
 282 +        * are in use
 283 +        */
 284 +       if (as->usbm)
 285 +               num_sgs = 0;
 286 +
 287         u += sizeof(struct async) + sizeof(struct urb) + uurb->buffer_length +
 288              num_sgs * sizeof(struct scatterlist);
 289         ret = usbfs_increase_memory_usage(u);
 290 @@ -1476,29 +1640,35 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
 291                         totlen -= u;
 292                 }
 293         } else if (uurb->buffer_length > 0) {
 294 -               as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
 295 -                               GFP_KERNEL);
 296 -               if (!as->urb->transfer_buffer) {
 297 -                       ret = -ENOMEM;
 298 -                       goto error;
 299 -               }
 300 +               if (as->usbm) {
 301 +                       unsigned long uurb_start = (unsigned long)uurb->buffer;
 302
 303 -               if (!is_in) {
 304 -                       if (copy_from_user(as->urb->transfer_buffer,
 305 -                                          uurb->buffer,
 306 -                                          uurb->buffer_length)) {
 307 -                               ret = -EFAULT;
 308 +                       as->urb->transfer_buffer = as->usbm->mem +
 309 +                                       (uurb_start - as->usbm->vm_start);
 310 +               } else {
 311 +                       as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
 312 +                                       GFP_KERNEL);
 313 +                       if (!as->urb->transfer_buffer) {
 314 +                               ret = -ENOMEM;
 315                                 goto error;
 316                         }
 317 -               } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
 318 -                       /*
 319 -                        * Isochronous input data may end up being
 320 -                        * discontiguous if some of the packets are short.
 321 -                        * Clear the buffer so that the gaps don't leak
 322 -                        * kernel data to userspace.
 323 -                        */
 324 -                       memset(as->urb->transfer_buffer, 0,
 325 -                                       uurb->buffer_length);
 326 +                       if (!is_in) {
 327 +                               if (copy_from_user(as->urb->transfer_buffer,
 328 +                                                  uurb->buffer,
 329 +                                                  uurb->buffer_length)) {
 330 +                                       ret = -EFAULT;
 331 +                                       goto error;
 332 +                               }
 333 +                       } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
 334 +                               /*
 335 +                                * Isochronous input data may end up being
 336 +                                * discontiguous if some of the packets are
 337 +                                * short. Clear the buffer so that the gaps
 338 +                                * don't leak kernel data to userspace.
 339 +                                */
 340 +                               memset(as->urb->transfer_buffer, 0,
 341 +                                               uurb->buffer_length);
 342 +                       }
 343                 }
 344         }
 345         as->urb->dev = ps->dev;
 346 @@ -1545,10 +1715,14 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
 347         isopkt = NULL;
 348         as->ps = ps;
 349         as->userurb = arg;
 350 -       if (is_in && uurb->buffer_length > 0)
 351 +       if (as->usbm) {
 352 +               unsigned long uurb_start = (unsigned long)uurb->buffer;
 353 +
 354 +               as->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
 355 +               as->urb->transfer_dma = as->usbm->dma_handle +
 356 +                               (uurb_start - as->usbm->vm_start);
 357 +       } else if (is_in && uurb->buffer_length > 0)
 358                 as->userbuffer = uurb->buffer;
 359 -       else
 360 -               as->userbuffer = NULL;
 361         as->signr = uurb->signr;
 362         as->ifnum = ifnum;
 363         as->pid = get_pid(task_pid(current));
 364 @@ -1604,6 +1778,8 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
 365         return 0;
 366
 367   error:
 368 +       if (as && as->usbm)
 369 +               dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
 370         kfree(isopkt);
 371         kfree(dr);
 372         if (as)
 373 @@ -2047,7 +2223,7 @@ static int proc_get_capabilities(struct usb_dev_state *ps, void __user *arg)
 374         __u32 caps;
 375
 376         caps = USBDEVFS_CAP_ZERO_PACKET | USBDEVFS_CAP_NO_PACKET_SIZE_LIM |
 377 -                       USBDEVFS_CAP_REAP_AFTER_DISCONNECT;
 378 +                       USBDEVFS_CAP_REAP_AFTER_DISCONNECT | USBDEVFS_CAP_MMAP;
 379         if (!ps->dev->bus->no_stop_on_short)
 380                 caps |= USBDEVFS_CAP_BULK_CONTINUATION;
 381         if (ps->dev->bus->sg_tablesize)
 382 @@ -2373,6 +2549,7 @@ const struct file_operations usbdev_file_operations = {
 383  #ifdef CONFIG_COMPAT
 384         .compat_ioctl =   usbdev_compat_ioctl,
 385  #endif
 386 +       .mmap =           usbdev_mmap,
 387         .open =           usbdev_open,
 388         .release =        usbdev_release,
 389  };
 390 diff --git a/include/uapi/linux/usbdevice_fs.h b/include/uapi/linux/usbdevice_fs.h
 391 index 019ba1e..ecbd176 100644
 392 --- a/include/uapi/linux/usbdevice_fs.h
 393 +++ b/include/uapi/linux/usbdevice_fs.h
 394 @@ -134,6 +134,7 @@ struct usbdevfs_hub_portinfo {
 395  #define USBDEVFS_CAP_NO_PACKET_SIZE_LIM                0x04
 396  #define USBDEVFS_CAP_BULK_SCATTER_GATHER       0x08
 397  #define USBDEVFS_CAP_REAP_AFTER_DISCONNECT     0x10
 398 +#define USBDEVFS_CAP_MMAP                      0x20
 399
 400  /* USBDEVFS_DISCONNECT_CLAIM flags & struct */
 401
 402 --
 403 2.1.4
 404