X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libbcachefs%2Futil.c;h=c50473d4925d9dd436b409fe2d8fc3a76f9dc4dd;hb=693e278c14efae893a6f051446fbbc0cf5e408cd;hp=e3ad26e244ab4cd161baf57d735dc022c386a343;hpb=b422ff58ba8eedcfef3b67b66468660f07b0cfc1;p=bcachefs-tools-debian diff --git a/libbcachefs/util.c b/libbcachefs/util.c index e3ad26e..c50473d 100644 --- a/libbcachefs/util.c +++ b/libbcachefs/util.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -21,22 +22,26 @@ #include #include #include +#include #include "eytzinger.h" #include "util.h" static const char si_units[] = "?kMGTPEZY"; -static int __bch2_strtoh(const char *cp, u64 *res, - u64 t_max, bool t_signed) +/* string_get_size units: */ +static const char *const units_2[] = { + "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB" +}; +static const char *const units_10[] = { + "B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB" +}; + +static int parse_u64(const char *cp, u64 *res) { - bool positive = *cp != '-'; - unsigned u; + const char *start = cp; u64 v = 0; - if (*cp == '+' || *cp == '-') - cp++; - if (!isdigit(*cp)) return -EINVAL; @@ -50,22 +55,122 @@ static int __bch2_strtoh(const char *cp, u64 *res, cp++; } while (isdigit(*cp)); + *res = v; + return cp - start; +} + +static int bch2_pow(u64 n, u64 p, u64 *res) +{ + *res = 1; + + while (p--) { + if (*res > div_u64(U64_MAX, n)) + return -ERANGE; + *res *= n; + } + return 0; +} + +static int parse_unit_suffix(const char *cp, u64 *res) +{ + const char *start = cp; + u64 base = 1024; + unsigned u; + int ret; + + if (*cp == ' ') + cp++; + for (u = 1; u < strlen(si_units); u++) if (*cp == si_units[u]) { cp++; goto got_unit; } - u = 0; + + for (u = 0; u < ARRAY_SIZE(units_2); u++) + if (!strncmp(cp, units_2[u], strlen(units_2[u]))) { + cp += strlen(units_2[u]); + goto got_unit; + } + + for (u = 0; u < ARRAY_SIZE(units_10); u++) + if (!strncmp(cp, units_10[u], strlen(units_10[u]))) { + cp += strlen(units_10[u]); + base = 1000; + goto got_unit; + } + + *res = 1; + return 0; got_unit: - if (*cp == '\n') + ret = bch2_pow(base, u, res); + if (ret) + return ret; + + return cp - start; +} + +#define parse_or_ret(cp, _f) \ +do { \ + int ret = _f; \ + if (ret < 0) \ + return ret; \ + cp += ret; \ +} while (0) + +static int __bch2_strtou64_h(const char *cp, u64 *res) +{ + const char *start = cp; + u64 v = 0, b, f_n = 0, f_d = 1; + int ret; + + parse_or_ret(cp, parse_u64(cp, &v)); + + if (*cp == '.') { cp++; - if (*cp) - return -EINVAL; + ret = parse_u64(cp, &f_n); + if (ret < 0) + return ret; + cp += ret; + + ret = bch2_pow(10, ret, &f_d); + if (ret) + return ret; + } + + parse_or_ret(cp, parse_unit_suffix(cp, &b)); + + if (v > div_u64(U64_MAX, b)) + return -ERANGE; + v *= b; + + if (f_n > div_u64(U64_MAX, b)) + return -ERANGE; - if (fls64(v) + u * 10 > 64) + f_n = div_u64(f_n * b, f_d); + if (v + f_n < v) return -ERANGE; + v += f_n; + + *res = v; + return cp - start; +} + +static int __bch2_strtoh(const char *cp, u64 *res, + u64 t_max, bool t_signed) +{ + bool positive = *cp != '-'; + u64 v = 0; + + if (*cp == '+' || *cp == '-') + cp++; - v <<= u * 10; + parse_or_ret(cp, __bch2_strtou64_h(cp, &v)); + + if (*cp == '\n') + cp++; + if (*cp) + return -EINVAL; if (positive) { if (v > t_max) @@ -86,7 +191,7 @@ got_unit: #define STRTO_H(name, type) \ int bch2_ ## name ## _h(const char *cp, type *res) \ { \ - u64 v; \ + u64 v = 0; \ int ret = __bch2_strtoh(cp, &v, ANYSINT_MAX(type), \ ANYSINT_MAX(type) != ((type) ~0ULL)); \ *res = v; \ @@ -99,58 +204,6 @@ STRTO_H(strtoll, long long) STRTO_H(strtoull, unsigned long long) STRTO_H(strtou64, u64) -void bch2_hprint(struct printbuf *buf, s64 v) -{ - int u, t = 0; - - for (u = 0; v >= 1024 || v <= -1024; u++) { - t = v & ~(~0U << 10); - v >>= 10; - } - - pr_buf(buf, "%lli", v); - - /* - * 103 is magic: t is in the range [-1023, 1023] and we want - * to turn it into [-9, 9] - */ - if (u && v < 100 && v > -100) - pr_buf(buf, ".%i", t / 103); - if (u) - pr_buf(buf, "%c", si_units[u]); -} - -void bch2_string_opt_to_text(struct printbuf *out, - const char * const list[], - size_t selected) -{ - size_t i; - - for (i = 0; list[i]; i++) - pr_buf(out, i == selected ? "[%s] " : "%s ", list[i]); -} - -void bch2_flags_to_text(struct printbuf *out, - const char * const list[], u64 flags) -{ - unsigned bit, nr = 0; - bool first = true; - - if (out->pos != out->end) - *out->pos = '\0'; - - while (list[nr]) - nr++; - - while (flags && (bit = __ffs(flags)) < nr) { - if (!first) - pr_buf(out, ","); - first = false; - pr_buf(out, "%s", list[bit]); - flags ^= 1 << bit; - } -} - u64 bch2_read_flag_list(char *opt, const char * const list[]) { u64 ret = 0; @@ -187,12 +240,85 @@ bool bch2_is_zero(const void *_p, size_t n) return true; } -static void bch2_quantiles_update(struct quantiles *q, u64 v) +void bch2_prt_u64_binary(struct printbuf *out, u64 v, unsigned nr_bits) +{ + while (nr_bits) + prt_char(out, '0' + ((v >> --nr_bits) & 1)); +} + +void bch2_print_string_as_lines(const char *prefix, const char *lines) +{ + const char *p; + + if (!lines) { + printk("%s (null)\n", prefix); + return; + } + + console_lock(); + while (1) { + p = strchrnul(lines, '\n'); + printk("%s%.*s\n", prefix, (int) (p - lines), lines); + if (!*p) + break; + lines = p + 1; + } + console_unlock(); +} + +int bch2_save_backtrace(bch_stacktrace *stack, struct task_struct *task) +{ + unsigned nr_entries = 0; + int ret = 0; + + stack->nr = 0; + ret = darray_make_room(stack, 32); + if (ret) + return ret; + + if (!down_read_trylock(&task->signal->exec_update_lock)) + return -1; + + do { + nr_entries = stack_trace_save_tsk(task, stack->data, stack->size, 0); + } while (nr_entries == stack->size && + !(ret = darray_make_room(stack, stack->size * 2))); + + stack->nr = nr_entries; + up_read(&task->signal->exec_update_lock); + + return ret; +} + +void bch2_prt_backtrace(struct printbuf *out, bch_stacktrace *stack) +{ + unsigned long *i; + + darray_for_each(*stack, i) { + prt_printf(out, "[<0>] %pB", (void *) *i); + prt_newline(out); + } +} + +int bch2_prt_task_backtrace(struct printbuf *out, struct task_struct *task) +{ + bch_stacktrace stack = { 0 }; + int ret = bch2_save_backtrace(&stack, task); + + bch2_prt_backtrace(out, &stack); + darray_exit(&stack); + return ret; +} + +/* time stats: */ + +#ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT +static void bch2_quantiles_update(struct bch2_quantiles *q, u64 v) { unsigned i = 0; while (i < ARRAY_SIZE(q->entries)) { - struct quantile_entry *e = q->entries + i; + struct bch2_quantile_entry *e = q->entries + i; if (unlikely(!e->step)) { e->m = v; @@ -217,85 +343,99 @@ static void bch2_quantiles_update(struct quantiles *q, u64 v) } } -/* time stats: */ - -static void bch2_time_stats_update_one(struct time_stats *stats, - u64 start, u64 end) +static inline void bch2_time_stats_update_one(struct bch2_time_stats *stats, + u64 start, u64 end) { u64 duration, freq; - duration = time_after64(end, start) - ? end - start : 0; - freq = time_after64(end, stats->last_event) - ? end - stats->last_event : 0; - - stats->count++; - - stats->average_duration = stats->average_duration - ? ewma_add(stats->average_duration, duration, 6) - : duration; + if (time_after64(end, start)) { + duration = end - start; + stats->duration_stats = mean_and_variance_update_inlined(stats->duration_stats, + duration); + stats->duration_stats_weighted = mean_and_variance_weighted_update( + stats->duration_stats_weighted, + duration); + stats->max_duration = max(stats->max_duration, duration); + stats->min_duration = min(stats->min_duration, duration); + bch2_quantiles_update(&stats->quantiles, duration); + } - stats->average_frequency = stats->average_frequency - ? ewma_add(stats->average_frequency, freq, 6) - : freq; + if (time_after64(end, stats->last_event)) { + freq = end - stats->last_event; + stats->freq_stats = mean_and_variance_update_inlined(stats->freq_stats, freq); + stats->freq_stats_weighted = mean_and_variance_weighted_update( + stats->freq_stats_weighted, + freq); + stats->max_freq = max(stats->max_freq, freq); + stats->min_freq = min(stats->min_freq, freq); + stats->last_event = end; + } +} - stats->max_duration = max(stats->max_duration, duration); +static noinline void bch2_time_stats_clear_buffer(struct bch2_time_stats *stats, + struct bch2_time_stat_buffer *b) +{ + struct bch2_time_stat_buffer_entry *i; + unsigned long flags; - stats->last_event = end; + spin_lock_irqsave(&stats->lock, flags); + for (i = b->entries; + i < b->entries + ARRAY_SIZE(b->entries); + i++) + bch2_time_stats_update_one(stats, i->start, i->end); + spin_unlock_irqrestore(&stats->lock, flags); - bch2_quantiles_update(&stats->quantiles, duration); + b->nr = 0; } -void __bch2_time_stats_update(struct time_stats *stats, u64 start, u64 end) +void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end) { unsigned long flags; + WARN_RATELIMIT(!stats->min_duration || !stats->min_freq, + "time_stats: min_duration = %llu, min_freq = %llu", + stats->min_duration, stats->min_freq); + if (!stats->buffer) { spin_lock_irqsave(&stats->lock, flags); bch2_time_stats_update_one(stats, start, end); - if (stats->average_frequency < 32 && - stats->count > 1024) + if (mean_and_variance_weighted_get_mean(stats->freq_stats_weighted) < 32 && + stats->duration_stats.n > 1024) stats->buffer = - alloc_percpu_gfp(struct time_stat_buffer, + alloc_percpu_gfp(struct bch2_time_stat_buffer, GFP_ATOMIC); spin_unlock_irqrestore(&stats->lock, flags); } else { - struct time_stat_buffer_entry *i; - struct time_stat_buffer *b; + struct bch2_time_stat_buffer *b; preempt_disable(); b = this_cpu_ptr(stats->buffer); BUG_ON(b->nr >= ARRAY_SIZE(b->entries)); - b->entries[b->nr++] = (struct time_stat_buffer_entry) { + b->entries[b->nr++] = (struct bch2_time_stat_buffer_entry) { .start = start, .end = end }; - if (b->nr == ARRAY_SIZE(b->entries)) { - spin_lock_irqsave(&stats->lock, flags); - for (i = b->entries; - i < b->entries + ARRAY_SIZE(b->entries); - i++) - bch2_time_stats_update_one(stats, i->start, i->end); - spin_unlock_irqrestore(&stats->lock, flags); - - b->nr = 0; - } - + if (unlikely(b->nr == ARRAY_SIZE(b->entries))) + bch2_time_stats_clear_buffer(stats, b); preempt_enable(); } } +#endif static const struct time_unit { const char *name; - u32 nsecs; + u64 nsecs; } time_units[] = { - { "ns", 1 }, - { "us", NSEC_PER_USEC }, - { "ms", NSEC_PER_MSEC }, - { "sec", NSEC_PER_SEC }, + { "ns", 1 }, + { "us", NSEC_PER_USEC }, + { "ms", NSEC_PER_MSEC }, + { "s", NSEC_PER_SEC }, + { "m", (u64) NSEC_PER_SEC * 60}, + { "h", (u64) NSEC_PER_SEC * 3600}, + { "eon", U64_MAX }, }; static const struct time_unit *pick_time_units(u64 ns) @@ -311,57 +451,153 @@ static const struct time_unit *pick_time_units(u64 ns) return u; } -static void pr_time_units(struct printbuf *out, u64 ns) +void bch2_pr_time_units(struct printbuf *out, u64 ns) { const struct time_unit *u = pick_time_units(ns); - pr_buf(out, "%llu %s", div_u64(ns, u->nsecs), u->name); + prt_printf(out, "%llu %s", div_u64(ns, u->nsecs), u->name); } -void bch2_time_stats_to_text(struct printbuf *out, struct time_stats *stats) +static void bch2_pr_time_units_aligned(struct printbuf *out, u64 ns) { - const struct time_unit *u; - u64 freq = READ_ONCE(stats->average_frequency); - u64 q, last_q = 0; - int i; + const struct time_unit *u = pick_time_units(ns); - pr_buf(out, "count:\t\t%llu\n", - stats->count); - pr_buf(out, "rate:\t\t%llu/sec\n", - freq ? div64_u64(NSEC_PER_SEC, freq) : 0); + prt_printf(out, "%llu ", div64_u64(ns, u->nsecs)); + prt_tab_rjust(out); + prt_printf(out, "%s", u->name); +} - pr_buf(out, "frequency:\t"); - pr_time_units(out, freq); +#define TABSTOP_SIZE 12 + +static inline void pr_name_and_units(struct printbuf *out, const char *name, u64 ns) +{ + prt_str(out, name); + prt_tab(out); + bch2_pr_time_units_aligned(out, ns); + prt_newline(out); +} - pr_buf(out, "\navg duration:\t"); - pr_time_units(out, stats->average_duration); +void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats) +{ + const struct time_unit *u; + s64 f_mean = 0, d_mean = 0; + u64 q, last_q = 0, f_stddev = 0, d_stddev = 0; + int i; + /* + * avoid divide by zero + */ + if (stats->freq_stats.n) { + f_mean = mean_and_variance_get_mean(stats->freq_stats); + f_stddev = mean_and_variance_get_stddev(stats->freq_stats); + d_mean = mean_and_variance_get_mean(stats->duration_stats); + d_stddev = mean_and_variance_get_stddev(stats->duration_stats); + } - pr_buf(out, "\nmax duration:\t"); - pr_time_units(out, stats->max_duration); + printbuf_tabstop_push(out, out->indent + TABSTOP_SIZE); + prt_printf(out, "count:"); + prt_tab(out); + prt_printf(out, "%llu ", + stats->duration_stats.n); + printbuf_tabstop_pop(out); + prt_newline(out); + + printbuf_tabstops_reset(out); + + printbuf_tabstop_push(out, out->indent + 20); + printbuf_tabstop_push(out, TABSTOP_SIZE + 2); + printbuf_tabstop_push(out, 0); + printbuf_tabstop_push(out, TABSTOP_SIZE + 2); + + prt_tab(out); + prt_printf(out, "since mount"); + prt_tab_rjust(out); + prt_tab(out); + prt_printf(out, "recent"); + prt_tab_rjust(out); + prt_newline(out); + + printbuf_tabstops_reset(out); + printbuf_tabstop_push(out, out->indent + 20); + printbuf_tabstop_push(out, TABSTOP_SIZE); + printbuf_tabstop_push(out, 2); + printbuf_tabstop_push(out, TABSTOP_SIZE); + + prt_printf(out, "duration of events"); + prt_newline(out); + printbuf_indent_add(out, 2); + + pr_name_and_units(out, "min:", stats->min_duration); + pr_name_and_units(out, "max:", stats->max_duration); + + prt_printf(out, "mean:"); + prt_tab(out); + bch2_pr_time_units_aligned(out, d_mean); + prt_tab(out); + bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_mean(stats->duration_stats_weighted)); + prt_newline(out); + + prt_printf(out, "stddev:"); + prt_tab(out); + bch2_pr_time_units_aligned(out, d_stddev); + prt_tab(out); + bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_stddev(stats->duration_stats_weighted)); + + printbuf_indent_sub(out, 2); + prt_newline(out); + + prt_printf(out, "time between events"); + prt_newline(out); + printbuf_indent_add(out, 2); + + pr_name_and_units(out, "min:", stats->min_freq); + pr_name_and_units(out, "max:", stats->max_freq); + + prt_printf(out, "mean:"); + prt_tab(out); + bch2_pr_time_units_aligned(out, f_mean); + prt_tab(out); + bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_mean(stats->freq_stats_weighted)); + prt_newline(out); + + prt_printf(out, "stddev:"); + prt_tab(out); + bch2_pr_time_units_aligned(out, f_stddev); + prt_tab(out); + bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_stddev(stats->freq_stats_weighted)); + + printbuf_indent_sub(out, 2); + prt_newline(out); + + printbuf_tabstops_reset(out); i = eytzinger0_first(NR_QUANTILES); u = pick_time_units(stats->quantiles.entries[i].m); - pr_buf(out, "\nquantiles (%s):\t", u->name); + prt_printf(out, "quantiles (%s):\t", u->name); eytzinger0_for_each(i, NR_QUANTILES) { bool is_last = eytzinger0_next(i, NR_QUANTILES) == -1; q = max(stats->quantiles.entries[i].m, last_q); - pr_buf(out, "%llu%s", - div_u64(q, u->nsecs), - is_last ? "\n" : " "); + prt_printf(out, "%llu ", + div_u64(q, u->nsecs)); + if (is_last) + prt_newline(out); last_q = q; } } -void bch2_time_stats_exit(struct time_stats *stats) +void bch2_time_stats_exit(struct bch2_time_stats *stats) { free_percpu(stats->buffer); } -void bch2_time_stats_init(struct time_stats *stats) +void bch2_time_stats_init(struct bch2_time_stats *stats) { memset(stats, 0, sizeof(*stats)); + stats->duration_stats_weighted.w = 8; + stats->freq_stats_weighted.w = 8; + stats->min_duration = U64_MAX; + stats->min_freq = U64_MAX; spin_lock_init(&stats->lock); } @@ -467,36 +703,45 @@ void bch2_pd_controller_init(struct bch_pd_controller *pd) pd->backpressure = 1; } -size_t bch2_pd_controller_print_debug(struct bch_pd_controller *pd, char *buf) +void bch2_pd_controller_debug_to_text(struct printbuf *out, struct bch_pd_controller *pd) { - /* 2^64 - 1 is 20 digits, plus null byte */ - char rate[21]; - char actual[21]; - char target[21]; - char proportional[21]; - char derivative[21]; - char change[21]; - s64 next_io; - - bch2_hprint(&PBUF(rate), pd->rate.rate); - bch2_hprint(&PBUF(actual), pd->last_actual); - bch2_hprint(&PBUF(target), pd->last_target); - bch2_hprint(&PBUF(proportional), pd->last_proportional); - bch2_hprint(&PBUF(derivative), pd->last_derivative); - bch2_hprint(&PBUF(change), pd->last_change); - - next_io = div64_s64(pd->rate.next - local_clock(), NSEC_PER_MSEC); - - return sprintf(buf, - "rate:\t\t%s/sec\n" - "target:\t\t%s\n" - "actual:\t\t%s\n" - "proportional:\t%s\n" - "derivative:\t%s\n" - "change:\t\t%s/sec\n" - "next io:\t%llims\n", - rate, target, actual, proportional, - derivative, change, next_io); + if (!out->nr_tabstops) + printbuf_tabstop_push(out, 20); + + prt_printf(out, "rate:"); + prt_tab(out); + prt_human_readable_s64(out, pd->rate.rate); + prt_newline(out); + + prt_printf(out, "target:"); + prt_tab(out); + prt_human_readable_u64(out, pd->last_target); + prt_newline(out); + + prt_printf(out, "actual:"); + prt_tab(out); + prt_human_readable_u64(out, pd->last_actual); + prt_newline(out); + + prt_printf(out, "proportional:"); + prt_tab(out); + prt_human_readable_s64(out, pd->last_proportional); + prt_newline(out); + + prt_printf(out, "derivative:"); + prt_tab(out); + prt_human_readable_s64(out, pd->last_derivative); + prt_newline(out); + + prt_printf(out, "change:"); + prt_tab(out); + prt_human_readable_s64(out, pd->last_change); + prt_newline(out); + + prt_printf(out, "next io:"); + prt_tab(out); + prt_printf(out, "%llims", div64_s64(pd->rate.next - local_clock(), NSEC_PER_MSEC)); + prt_newline(out); } /* misc: */ @@ -516,16 +761,20 @@ void bch2_bio_map(struct bio *bio, void *base, size_t size) } } -int bch2_bio_alloc_pages(struct bio *bio, size_t size, gfp_t gfp_mask) +int _bch2_bio_alloc_pages(struct bio *bio, size_t size, gfp_t gfp_mask) { while (size) { - struct page *page = alloc_page(gfp_mask); + struct page *page = _alloc_pages(gfp_mask, 0); unsigned len = min_t(size_t, PAGE_SIZE, size); if (!page) return -ENOMEM; - BUG_ON(!bio_add_page(bio, page, len, 0)); + if (unlikely(!bio_add_page(bio, page, len, 0))) { + __free_page(page); + break; + } + size -= len; } @@ -575,21 +824,6 @@ void memcpy_from_bio(void *dst, struct bio *src, struct bvec_iter src_iter) } } -void bch_scnmemcpy(struct printbuf *out, - const char *src, size_t len) -{ - size_t n = printbuf_remaining(out); - - if (n) { - n = min(n - 1, len); - memcpy(out->pos, src, n); - out->pos += n; - *out->pos = '\0'; - } -} - -#include "eytzinger.h" - static int alignment_ok(const void *base, size_t align) { return IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || @@ -890,6 +1124,7 @@ u64 *bch2_acc_percpu_u64s(u64 __percpu *p, unsigned nr) u64 *ret; int cpu; + /* access to pcpu vars has to be blocked by other locking */ preempt_disable(); ret = this_cpu_ptr(p); preempt_enable();