13 #ifdef CONFIG_VALGRIND
14 #include <valgrind/memcheck.h>
17 #include <linux/bio.h>
18 #include <linux/blkdev.h>
19 #include <linux/completion.h>
21 #include <linux/kthread.h>
23 #include "tools-util.h"
25 static io_context_t aio_ctx;
26 static atomic_t running_requests;
28 void generic_make_request(struct bio *bio)
31 struct bvec_iter iter;
36 if (bio->bi_opf & REQ_PREFLUSH) {
37 ret = fdatasync(bio->bi_bdev->bd_fd);
39 fprintf(stderr, "fsync error: %m\n");
40 bio->bi_status = BLK_STS_IOERR;
47 bio_for_each_segment(bv, bio, iter)
50 iov = alloca(sizeof(*iov) * i);
53 bio_for_each_segment(bv, bio, iter) {
54 void *start = page_address(bv.bv_page) + bv.bv_offset;
55 size_t len = bv.bv_len;
57 iov[i++] = (struct iovec) {
62 #ifdef CONFIG_VALGRIND
63 /* To be pedantic it should only be on IO completion. */
64 if (bio_op(bio) == REQ_OP_READ)
65 VALGRIND_MAKE_MEM_DEFINED(start, len);
71 .aio_fildes = bio->bi_opf & REQ_FUA
72 ? bio->bi_bdev->bd_sync_fd
73 : bio->bi_bdev->bd_fd,
76 switch (bio_op(bio)) {
78 iocb.aio_lio_opcode = IO_CMD_PREADV;
81 iocb.u.v.offset = bio->bi_iter.bi_sector << 9;
83 atomic_inc(&running_requests);
84 ret = io_submit(aio_ctx, 1, &iocbp);
86 die("io_submit err: %s", strerror(-ret));
89 iocb.aio_lio_opcode = IO_CMD_PWRITEV;
92 iocb.u.v.offset = bio->bi_iter.bi_sector << 9;
94 atomic_inc(&running_requests);
95 ret = io_submit(aio_ctx, 1, &iocbp);
97 die("io_submit err: %s", strerror(-ret));
100 ret = fsync(bio->bi_bdev->bd_fd);
102 die("fsync error: %m");
110 static void submit_bio_wait_endio(struct bio *bio)
112 complete(bio->bi_private);
115 int submit_bio_wait(struct bio *bio)
117 struct completion done;
119 init_completion(&done);
120 bio->bi_private = &done;
121 bio->bi_end_io = submit_bio_wait_endio;
122 bio->bi_opf |= REQ_SYNC;
124 wait_for_completion(&done);
126 return blk_status_to_errno(bio->bi_status);
129 int blkdev_issue_discard(struct block_device *bdev,
130 sector_t sector, sector_t nr_sects,
131 gfp_t gfp_mask, unsigned long flags)
136 unsigned bdev_logical_block_size(struct block_device *bdev)
142 ret = fstat(bdev->bd_fd, &statbuf);
145 if (!S_ISBLK(statbuf.st_mode))
146 return statbuf.st_blksize >> 9;
148 ret = ioctl(bdev->bd_fd, BLKPBSZGET, &blksize);
154 sector_t get_capacity(struct gendisk *disk)
156 struct block_device *bdev =
157 container_of(disk, struct block_device, __bd_disk);
162 ret = fstat(bdev->bd_fd, &statbuf);
165 if (!S_ISBLK(statbuf.st_mode))
166 return statbuf.st_size >> 9;
168 ret = ioctl(bdev->bd_fd, BLKGETSIZE64, &bytes);
174 void blkdev_put(struct block_device *bdev, fmode_t mode)
176 fdatasync(bdev->bd_fd);
177 close(bdev->bd_sync_fd);
182 struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
185 struct block_device *bdev;
186 int fd, sync_fd, flags = O_DIRECT;
188 if ((mode & (FMODE_READ|FMODE_WRITE)) == (FMODE_READ|FMODE_WRITE))
190 else if (mode & FMODE_READ)
192 else if (mode & FMODE_WRITE)
196 /* using O_EXCL doesn't work with opening twice for an O_SYNC fd: */
197 if (mode & FMODE_EXCL)
201 fd = open(path, flags);
203 return ERR_PTR(-errno);
205 sync_fd = open(path, flags|O_SYNC);
209 return ERR_PTR(-errno);
212 bdev = malloc(sizeof(*bdev));
213 memset(bdev, 0, sizeof(*bdev));
215 strncpy(bdev->name, path, sizeof(bdev->name));
216 bdev->name[sizeof(bdev->name) - 1] = '\0';
218 bdev->bd_dev = xfstat(fd).st_rdev;
220 bdev->bd_sync_fd = sync_fd;
221 bdev->bd_holder = holder;
222 bdev->bd_disk = &bdev->__bd_disk;
223 bdev->bd_bdi = &bdev->__bd_bdi;
224 bdev->queue.backing_dev_info = bdev->bd_bdi;
229 void bdput(struct block_device *bdev)
234 int lookup_bdev(const char *path, dev_t *dev)
239 static int aio_completion_thread(void *arg)
241 struct io_event events[8], *ev;
246 ret = io_getevents(aio_ctx, 1, ARRAY_SIZE(events),
249 if (ret < 0 && ret == -EINTR)
252 die("io_getevents() error: %s", strerror(-ret));
254 for (ev = events; ev < events + ret; ev++) {
255 struct bio *bio = (struct bio *) ev->data;
257 /* This should only happen during blkdev_cleanup() */
259 BUG_ON(atomic_read(&running_requests) != 0);
264 if (ev->res != bio->bi_iter.bi_size)
265 bio->bi_status = BLK_STS_IOERR;
268 atomic_dec(&running_requests);
275 static struct task_struct *aio_task = NULL;
277 __attribute__((constructor(102)))
278 static void blkdev_init(void)
280 struct task_struct *p;
282 if (io_setup(256, &aio_ctx))
283 die("io_setup() error: %m");
285 p = kthread_run(aio_completion_thread, NULL, "aio_completion");
291 __attribute__((destructor(102)))
292 static void blkdev_cleanup(void)
294 struct task_struct *p = NULL;
298 /* I mean, really?! IO_CMD_NOOP is even defined, but not implemented. */
302 die("pipe err: %s", strerror(ret));
304 /* Wake up the completion thread with spurious work. */
307 .aio_lio_opcode = IO_CMD_PWRITE,
308 .data = NULL, /* Signal to stop */
309 .aio_fildes = fds[1],
313 ret = io_submit(aio_ctx, 1, &iocbp);
315 die("io_submit cleanup err: %s", strerror(-ret));
317 ret = kthread_stop(p);