1 // SPDX-License-Identifier: GPL-2.0
5 #include <linux/freezer.h>
6 #include <linux/kthread.h>
7 #include <linux/preempt.h>
9 static inline long io_timer_cmp(io_timer_heap *h,
13 return l->expire - r->expire;
16 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer)
20 spin_lock(&clock->timer_lock);
22 if (time_after_eq((unsigned long) atomic64_read(&clock->now),
24 spin_unlock(&clock->timer_lock);
29 for (i = 0; i < clock->timers.used; i++)
30 if (clock->timers.data[i] == timer)
33 BUG_ON(!heap_add(&clock->timers, timer, io_timer_cmp, NULL));
35 spin_unlock(&clock->timer_lock);
38 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer)
42 spin_lock(&clock->timer_lock);
44 for (i = 0; i < clock->timers.used; i++)
45 if (clock->timers.data[i] == timer) {
46 heap_del(&clock->timers, i, io_timer_cmp, NULL);
50 spin_unlock(&clock->timer_lock);
53 struct io_clock_wait {
54 struct io_timer io_timer;
55 struct timer_list cpu_timer;
56 struct task_struct *task;
60 static void io_clock_wait_fn(struct io_timer *timer)
62 struct io_clock_wait *wait = container_of(timer,
63 struct io_clock_wait, io_timer);
66 wake_up_process(wait->task);
69 static void io_clock_cpu_timeout(struct timer_list *timer)
71 struct io_clock_wait *wait = container_of(timer,
72 struct io_clock_wait, cpu_timer);
75 wake_up_process(wait->task);
78 void bch2_io_clock_schedule_timeout(struct io_clock *clock, unsigned long until)
80 struct io_clock_wait wait;
82 /* XXX: calculate sleep time rigorously */
83 wait.io_timer.expire = until;
84 wait.io_timer.fn = io_clock_wait_fn;
87 bch2_io_timer_add(clock, &wait.io_timer);
91 bch2_io_timer_del(clock, &wait.io_timer);
94 void bch2_kthread_io_clock_wait(struct io_clock *clock,
95 unsigned long io_until,
96 unsigned long cpu_timeout)
98 struct io_clock_wait wait;
100 wait.io_timer.expire = io_until;
101 wait.io_timer.fn = io_clock_wait_fn;
104 bch2_io_timer_add(clock, &wait.io_timer);
106 timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0);
108 if (cpu_timeout != MAX_SCHEDULE_TIMEOUT)
109 mod_timer(&wait.cpu_timer, cpu_timeout + jiffies);
112 set_current_state(TASK_INTERRUPTIBLE);
113 if (kthread_should_stop())
123 __set_current_state(TASK_RUNNING);
124 del_timer_sync(&wait.cpu_timer);
125 destroy_timer_on_stack(&wait.cpu_timer);
126 bch2_io_timer_del(clock, &wait.io_timer);
129 static struct io_timer *get_expired_timer(struct io_clock *clock,
132 struct io_timer *ret = NULL;
134 spin_lock(&clock->timer_lock);
136 if (clock->timers.used &&
137 time_after_eq(now, clock->timers.data[0]->expire))
138 heap_pop(&clock->timers, ret, io_timer_cmp, NULL);
140 spin_unlock(&clock->timer_lock);
145 void __bch2_increment_clock(struct io_clock *clock, unsigned sectors)
147 struct io_timer *timer;
148 unsigned long now = atomic64_add_return(sectors, &clock->now);
150 while ((timer = get_expired_timer(clock, now)))
154 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock)
160 spin_lock(&clock->timer_lock);
161 now = atomic64_read(&clock->now);
163 for (i = 0; i < clock->timers.used; i++)
164 prt_printf(out, "%ps:\t%li\n",
165 clock->timers.data[i]->fn,
166 clock->timers.data[i]->expire - now);
167 spin_unlock(&clock->timer_lock);
171 void bch2_io_clock_exit(struct io_clock *clock)
173 free_heap(&clock->timers);
174 free_percpu(clock->pcpu_buf);
177 int bch2_io_clock_init(struct io_clock *clock)
179 atomic64_set(&clock->now, 0);
180 spin_lock_init(&clock->timer_lock);
182 clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus();
184 clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf);
185 if (!clock->pcpu_buf)
186 return -BCH_ERR_ENOMEM_io_clock_init;
188 if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL))
189 return -BCH_ERR_ENOMEM_io_clock_init;