Use aio

diff --git a/Makefile b/Makefile
index 327fec25fe248fe4ff79d6cd31fbeb065edd0cd8..d6286dea624edb0a1f709eb29a39eef416041d81 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -36,7 +36,7 @@ endif
 PKGCONFIG_LIBS="blkid uuid liburcu libsodium zlib"
 CFLAGS+=`pkg-config --cflags   ${PKGCONFIG_LIBS}`
 LDLIBS+=`pkg-config --libs     ${PKGCONFIG_LIBS}`              \
-       -lm -lpthread -lrt -lscrypt -lkeyutils
+       -lm -lpthread -lrt -lscrypt -lkeyutils -laio
 
 ifeq ($(PREFIX),/usr)
        ROOT_SBINDIR=/sbin

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index eb15726991c2d6fd3b708d17800f1af595f75a5e..4b06a0ccd1b0cead9245ff6f940485e141306694 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -81,6 +81,7 @@ struct block_device {
        struct gendisk          *bd_disk;
        struct gendisk          __bd_disk;
        int                     bd_fd;
+       int                     bd_sync_fd;
 };
 
 void generic_make_request(struct bio *);

diff --git a/linux/blkdev.c b/linux/blkdev.c
index c72df60fdf10e8bb1a605bb9a7b88c34780fa46b..c0c8fd0d0d8665e030b834e939ab37dfed8196bd 100644 (file)
--- a/linux/blkdev.c
+++ b/linux/blkdev.c
@@ -8,11 +8,19 @@
 #include <sys/uio.h>
 #include <unistd.h>
 
+#include <libaio.h>
+
 #include <linux/bio.h>
 #include <linux/blkdev.h>
+#include <linux/completion.h>
 #include <linux/fs.h>
+#include <linux/kthread.h>
 
-int submit_bio_wait(struct bio *bio)
+#include "tools-util.h"
+
+static io_context_t aio_ctx;
+
+void generic_make_request(struct bio *bio)
 {
        struct iovec *iov;
        struct bvec_iter iter;
@@ -24,7 +32,9 @@ int submit_bio_wait(struct bio *bio)
                ret = fdatasync(bio->bi_bdev->bd_fd);
                if (ret) {
                        fprintf(stderr, "fsync error: %m\n");
-                       return -EIO;
+                       bio->bi_error = -EIO;
+                       bio_endio(bio);
+                       return;
                }
        }
 
@@ -41,39 +51,54 @@ int submit_bio_wait(struct bio *bio)
                        .iov_len = bv.bv_len,
                };
 
+       struct iocb iocb = {
+               .data           = bio,
+               .aio_fildes     = bio->bi_opf & REQ_FUA
+                       ? bio->bi_bdev->bd_sync_fd
+                       : bio->bi_bdev->bd_fd,
+       }, *iocbp = &iocb;
+
        switch (bio_op(bio)) {
        case REQ_OP_READ:
-               ret = preadv(bio->bi_bdev->bd_fd, iov, i,
-                            bio->bi_iter.bi_sector << 9);
+               iocb.aio_lio_opcode     = IO_CMD_PREADV;
+               iocb.u.v.vec            = iov;
+               iocb.u.v.nr             = i;
+               iocb.u.v.offset         = bio->bi_iter.bi_sector << 9;
+
+               if (io_submit(aio_ctx, 1, &iocbp) != 1)
+                       die("io_submit err: %m");
                break;
        case REQ_OP_WRITE:
-               ret = pwritev(bio->bi_bdev->bd_fd, iov, i,
-                             bio->bi_iter.bi_sector << 9);
+               iocb.aio_lio_opcode     = IO_CMD_PWRITEV;
+               iocb.u.v.vec            = iov;
+               iocb.u.v.nr             = i;
+               iocb.u.v.offset         = bio->bi_iter.bi_sector << 9;
+
+               if (io_submit(aio_ctx, 1, &iocbp) != 1)
+                       die("io_submit err: %m");
                break;
        default:
                BUG();
        }
+}
 
-       if (ret != bio->bi_iter.bi_size) {
-               fprintf(stderr, "IO error: %li (%m)\n", ret);
-               return -EIO;
-       }
-
-       if (bio->bi_opf & REQ_FUA) {
-               ret = fdatasync(bio->bi_bdev->bd_fd);
-               if (ret) {
-                       fprintf(stderr, "fsync error: %m\n");
-                       return -EIO;
-               }
-       }
-
-       return 0;
+static void submit_bio_wait_endio(struct bio *bio)
+{
+       complete(bio->bi_private);
 }
 
-void generic_make_request(struct bio *bio)
+int submit_bio_wait(struct bio *bio)
 {
-       bio->bi_error = submit_bio_wait(bio);
-       bio_endio(bio);
+       struct completion done;
+
+       init_completion(&done);
+       bio->bi_private = &done;
+       bio->bi_end_io = submit_bio_wait_endio;
+       bio->bi_opf |= REQ_SYNC;
+       submit_bio(bio);
+       wait_for_completion(&done);
+
+       return bio->bi_error;
 }
 
 int blkdev_issue_discard(struct block_device *bdev,
@@ -124,6 +149,7 @@ sector_t get_capacity(struct gendisk *disk)
 void blkdev_put(struct block_device *bdev, fmode_t mode)
 {
        fdatasync(bdev->bd_fd);
+       close(bdev->bd_sync_fd);
        close(bdev->bd_fd);
        free(bdev);
 }
@@ -132,7 +158,7 @@ struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
                                        void *holder)
 {
        struct block_device *bdev;
-       int fd, flags = O_DIRECT;
+       int fd, sync_fd, flags = O_DIRECT;
 
        if ((mode & (FMODE_READ|FMODE_WRITE)) == (FMODE_READ|FMODE_WRITE))
                flags = O_RDWR;
@@ -148,15 +174,22 @@ struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
        if (fd < 0)
                return ERR_PTR(-errno);
 
+       sync_fd = open(path, flags|O_SYNC);
+       if (fd < 0) {
+               close(fd);
+               return ERR_PTR(-errno);
+       }
+
        bdev = malloc(sizeof(*bdev));
        memset(bdev, 0, sizeof(*bdev));
 
        strncpy(bdev->name, path, sizeof(bdev->name));
        bdev->name[sizeof(bdev->name) - 1] = '\0';
 
-       bdev->bd_fd     = fd;
-       bdev->bd_holder = holder;
-       bdev->bd_disk   = &bdev->__bd_disk;
+       bdev->bd_fd             = fd;
+       bdev->bd_sync_fd        = sync_fd;
+       bdev->bd_holder         = holder;
+       bdev->bd_disk           = &bdev->__bd_disk;
 
        return bdev;
 }
@@ -170,3 +203,44 @@ struct block_device *lookup_bdev(const char *path)
 {
        return ERR_PTR(-EINVAL);
 }
+
+static int aio_completion_thread(void *arg)
+{
+       struct io_event events[8], *ev;
+       int ret;
+
+       while (1) {
+               ret = io_getevents(aio_ctx, 1, ARRAY_SIZE(events),
+                                  events, NULL);
+
+               if (ret < 0 && errno == EINTR)
+                       continue;
+               if (ret < 0)
+                       die("io_getevents() error: %m");
+
+               for (ev = events; ev < events + ret; ev++) {
+                       struct bio *bio = (struct bio *) ev->data;
+
+                       if (ev->res < 0)
+                               bio->bi_error = ev->res;
+                       else if (ev->res != bio->bi_iter.bi_size)
+                               bio->bi_error = -EIO;
+
+                       bio_endio(bio);
+               }
+       }
+
+       return 0;
+}
+
+__attribute__((constructor(102)))
+static void blkdev_init(void)
+{
+       struct task_struct *p;
+
+       if (io_setup(256, &aio_ctx))
+               die("io_setup() error: %m");
+
+       p = kthread_run(aio_completion_thread, NULL, "aio_completion");
+       BUG_ON(IS_ERR(p));
+}