X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libavfilter%2Fvf_readeia608.c;h=9558e938ab813914bc14c62a63a8c002b1659005;hb=a859e57424ca0a00d17bdc034c52005c18184961;hp=bc3abe7c4d25e781f6e86cb9f10e335ed5870757;hpb=b8eb0827f053230dd919bc73e25381b47b4fe1a7;p=ffmpeg diff --git a/libavfilter/vf_readeia608.c b/libavfilter/vf_readeia608.c index bc3abe7c4d2..9558e938ab8 100644 --- a/libavfilter/vf_readeia608.c +++ b/libavfilter/vf_readeia608.c @@ -36,38 +36,65 @@ #include "internal.h" #include "video.h" -#define FALL 0 -#define RISE 1 +#define LAG 25 +#define CLOCK_BITSIZE_MIN 0.2f +#define CLOCK_BITSIZE_MAX 1.5f +#define SYNC_BITSIZE_MIN 12.f +#define SYNC_BITSIZE_MAX 15.f + +typedef struct LineItem { + int input; + int output; + + float unfiltered; + float filtered; + float average; + float deviation; +} LineItem; + +typedef struct CodeItem { + uint8_t bit; + int size; +} CodeItem; + +typedef struct ScanItem { + int nb_line; + int found; + int white; + int black; + uint64_t *histogram; + uint8_t byte[2]; + + CodeItem *code; + LineItem *line; +} ScanItem; typedef struct ReadEIA608Context { const AVClass *class; + int start, end; - int min_range; - int max_peak_diff; - int max_period_diff; - int max_start_diff; - int nb_found; - int white; - int black; - float mpd, mhd, msd, mac, spw, bhd, wth, bth; + float spw; int chp; + int lp; + + int depth; + int max; + int nb_allocated; + ScanItem *scan; + + void (*read_line[2])(AVFrame *in, int nb_line, + LineItem *line, int lp, int w); } ReadEIA608Context; #define OFFSET(x) offsetof(ReadEIA608Context, x) -#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM +#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_RUNTIME_PARAM static const AVOption readeia608_options[] = { - { "scan_min", "set from which line to scan for codes", OFFSET(start), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX, FLAGS }, - { "scan_max", "set to which line to scan for codes", OFFSET(end), AV_OPT_TYPE_INT, {.i64=29}, 0, INT_MAX, FLAGS }, - { "mac", "set minimal acceptable amplitude change for sync codes detection", OFFSET(mac), AV_OPT_TYPE_FLOAT, {.dbl=.2}, 0.001, 1, FLAGS }, - { "spw", "set ratio of width reserved for sync code detection", OFFSET(spw), AV_OPT_TYPE_FLOAT, {.dbl=.27}, 0.1, 0.7, FLAGS }, - { "mhd", "set max peaks height difference for sync code detection", OFFSET(mhd), AV_OPT_TYPE_FLOAT, {.dbl=.1}, 0, 0.5, FLAGS }, - { "mpd", "set max peaks period difference for sync code detection", OFFSET(mpd), AV_OPT_TYPE_FLOAT, {.dbl=.1}, 0, 0.5, FLAGS }, - { "msd", "set first two max start code bits differences", OFFSET(msd), AV_OPT_TYPE_FLOAT, {.dbl=.02}, 0, 0.5, FLAGS }, - { "bhd", "set min ratio of bits height compared to 3rd start code bit", OFFSET(bhd), AV_OPT_TYPE_FLOAT, {.dbl=.75}, 0.01, 1, FLAGS }, - { "th_w", "set white color threshold", OFFSET(wth), AV_OPT_TYPE_FLOAT, {.dbl=.35}, 0.1, 1, FLAGS }, - { "th_b", "set black color threshold", OFFSET(bth), AV_OPT_TYPE_FLOAT, {.dbl=.15}, 0, 0.5, FLAGS }, - { "chp", "check and apply parity bit", OFFSET(chp), AV_OPT_TYPE_BOOL, {.i64= 0}, 0, 1, FLAGS }, + { "scan_min", "set from which line to scan for codes", OFFSET(start), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX, FLAGS }, + { "scan_max", "set to which line to scan for codes", OFFSET(end), AV_OPT_TYPE_INT, {.i64=29}, 0, INT_MAX, FLAGS }, + { "spw", "set ratio of width reserved for sync code detection", OFFSET(spw), AV_OPT_TYPE_FLOAT, {.dbl=.27}, 0.1, 0.7, FLAGS }, + { "chp", "check and apply parity bit", OFFSET(chp), AV_OPT_TYPE_BOOL, {.i64= 0}, 0, 1, FLAGS }, + { "lp", "lowpass line prior to processing", OFFSET(lp), AV_OPT_TYPE_BOOL, {.i64= 1}, 0, 1, FLAGS }, { NULL } }; @@ -76,13 +103,26 @@ AVFILTER_DEFINE_CLASS(readeia608); static int query_formats(AVFilterContext *ctx) { static const enum AVPixelFormat pixel_fmts[] = { - AV_PIX_FMT_GRAY8, + AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9, + AV_PIX_FMT_GRAY10, AV_PIX_FMT_GRAY12, AV_PIX_FMT_GRAY14, + AV_PIX_FMT_GRAY16, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P, AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVJ411P, + AV_PIX_FMT_YUV420P9, AV_PIX_FMT_YUV422P9, AV_PIX_FMT_YUV444P9, + AV_PIX_FMT_YUV420P10, AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10, + AV_PIX_FMT_YUV440P10, + AV_PIX_FMT_YUV444P12, AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P12, + AV_PIX_FMT_YUV440P12, + AV_PIX_FMT_YUV444P14, AV_PIX_FMT_YUV422P14, AV_PIX_FMT_YUV420P14, + AV_PIX_FMT_YUV420P16, AV_PIX_FMT_YUV422P16, AV_PIX_FMT_YUV444P16, + AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P, + AV_PIX_FMT_YUVA444P9, AV_PIX_FMT_YUVA444P10, AV_PIX_FMT_YUVA444P12, AV_PIX_FMT_YUVA444P16, + AV_PIX_FMT_YUVA422P9, AV_PIX_FMT_YUVA422P10, AV_PIX_FMT_YUVA422P12, AV_PIX_FMT_YUVA422P16, + AV_PIX_FMT_YUVA420P9, AV_PIX_FMT_YUVA420P10, AV_PIX_FMT_YUVA420P16, AV_PIX_FMT_NONE }; AVFilterFormats *formats = ff_make_format_list(pixel_fmts); @@ -91,137 +131,340 @@ static int query_formats(AVFilterContext *ctx) return ff_set_common_formats(ctx, formats); } -static int config_input(AVFilterLink *inlink) +static int config_filter(AVFilterContext *ctx, int start, int end) { - const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); - AVFilterContext *ctx = inlink->dst; ReadEIA608Context *s = ctx->priv; - int depth = desc->comp[0].depth; + AVFilterLink *inlink = ctx->inputs[0]; + int size = inlink->w + LAG; - if (s->end >= inlink->h) { + if (end >= inlink->h) { av_log(ctx, AV_LOG_WARNING, "Last line to scan too large, clipping.\n"); - s->end = inlink->h - 1; + end = inlink->h - 1; } - if (s->start > s->end) { + if (start > end) { av_log(ctx, AV_LOG_ERROR, "Invalid range.\n"); return AVERROR(EINVAL); } - s->min_range = s->mac * ((1 << depth) - 1); - s->max_peak_diff = s->mhd * ((1 << depth) - 1); - s->max_period_diff = s->mpd * ((1 << depth) - 1); - s->max_start_diff = s->msd * ((1 << depth) - 1); - s->white = s->wth * ((1 << depth) - 1); - s->black = s->bth * ((1 << depth) - 1); + if (s->nb_allocated < end - start + 1) { + const int diff = end - start + 1 - s->nb_allocated; + + s->scan = av_realloc_f(s->scan, end - start + 1, sizeof(*s->scan)); + if (!s->scan) + return AVERROR(ENOMEM); + memset(&s->scan[s->nb_allocated], 0, diff * sizeof(*s->scan)); + s->nb_allocated = end - start + 1; + } + + for (int i = 0; i < s->nb_allocated; i++) { + ScanItem *scan = &s->scan[i]; + + if (!scan->histogram) + scan->histogram = av_calloc(s->max + 1, sizeof(*scan->histogram)); + if (!scan->line) + scan->line = av_calloc(size, sizeof(*scan->line)); + if (!scan->code) + scan->code = av_calloc(size, sizeof(*scan->code)); + if (!scan->line || !scan->code || !scan->histogram) + return AVERROR(ENOMEM); + } + + s->start = start; + s->end = end; return 0; } -static void extract_line(AVFilterContext *ctx, AVFilterLink *inlink, AVFrame *in, int line) +static void build_histogram(ReadEIA608Context *s, ScanItem *scan, const LineItem *line, int len) { - ReadEIA608Context *s = ctx->priv; - int max = 0, min = INT_MAX; - int i, ch, range = 0; - const uint8_t *src; - uint16_t clock[8][2] = { { 0 } }; - const int sync_width = s->spw * in->width; - int last = 0, peaks = 0, max_peak_diff = 0, dir = RISE; - const int width_per_bit = (in->width - sync_width) / 19; - uint8_t byte[2] = { 0 }; - int s1, s2, s3, parity; - - src = &in->data[0][line * in->linesize[0]]; - for (i = 0; i < sync_width; i++) { - max = FFMAX(max, src[i]); - min = FFMIN(min, src[i]); + memset(scan->histogram, 0, (s->max + 1) * sizeof(*scan->histogram)); + + for (int i = LAG; i < len + LAG; i++) + scan->histogram[line[i].input]++; +} + +static void find_black_and_white(ReadEIA608Context *s, ScanItem *scan) +{ + const int max = s->max; + int start = 0, end = 0, middle; + int black = 0, white = 0; + int cnt; + + for (int i = 0; i <= max; i++) { + if (scan->histogram[i]) { + start = i; + break; + } } - range = max - min; - if (range < s->min_range) - return; + for (int i = max; i >= 0; i--) { + if (scan->histogram[i]) { + end = i; + break; + } + } - for (i = 0; i < sync_width; i++) { - int Y = src[i]; - - if (dir == RISE) { - if (Y < last) { - dir = FALL; - if (last >= s->white) { - clock[peaks][0] = last; - clock[peaks][1] = i; - peaks++; - if (peaks > 7) - break; - } - } - } else if (dir == FALL) { - if (Y > last && last <= s->black) { - dir = RISE; + middle = start + (end - start) / 2; + + cnt = 0; + for (int i = start; i <= middle; i++) { + if (scan->histogram[i] > cnt) { + cnt = scan->histogram[i]; + black = i; + } + } + + cnt = 0; + for (int i = end; i >= middle; i--) { + if (scan->histogram[i] > cnt) { + cnt = scan->histogram[i]; + white = i; + } + } + + scan->black = black; + scan->white = white; +} + +static float meanf(const LineItem *line, int len) +{ + float sum = 0.0, mean = 0.0; + + for (int i = 0; i < len; i++) + sum += line[i].filtered; + + mean = sum / len; + + return mean; +} + +static float stddevf(const LineItem *line, int len) +{ + float m = meanf(line, len); + float standard_deviation = 0.f; + + for (int i = 0; i < len; i++) + standard_deviation += (line[i].filtered - m) * (line[i].filtered - m); + + return sqrtf(standard_deviation / (len - 1)); +} + +static void thresholding(ReadEIA608Context *s, ScanItem *scan, LineItem *line, + int lag, float threshold, float influence, int len) +{ + for (int i = lag; i < len + lag; i++) { + line[i].unfiltered = line[i].input / 255.f; + line[i].filtered = line[i].unfiltered; + } + + for (int i = 0; i < lag; i++) { + line[i].unfiltered = meanf(line, len * s->spw); + line[i].filtered = line[i].unfiltered; + } + + line[lag - 1].average = meanf(line, lag); + line[lag - 1].deviation = stddevf(line, lag); + + for (int i = lag; i < len + lag; i++) { + if (fabsf(line[i].unfiltered - line[i-1].average) > threshold * line[i-1].deviation) { + if (line[i].unfiltered > line[i-1].average) { + line[i].output = 255; + } else { + line[i].output = 0; } + + line[i].filtered = influence * line[i].unfiltered + (1.f - influence) * line[i-1].filtered; + } else { + int distance_from_black, distance_from_white; + + distance_from_black = FFABS(line[i].input - scan->black); + distance_from_white = FFABS(line[i].input - scan->white); + + line[i].output = distance_from_black <= distance_from_white ? 0 : 255; } - last = Y; + + line[i].average = meanf(line + i - lag, lag); + line[i].deviation = stddevf(line + i - lag, lag); } +} - if (peaks != 7) - return; +static int periods(const LineItem *line, CodeItem *code, int len) +{ + int hold = line[LAG].output, cnt = 0; + int last = LAG; + + memset(code, 0, len * sizeof(*code)); + + for (int i = LAG + 1; i < len + LAG; i++) { + if (line[i].output != hold) { + code[cnt].size = i - last; + code[cnt].bit = hold; + hold = line[i].output; + last = i; + cnt++; + } + } - for (i = 1; i < 7; i++) - max_peak_diff = FFMAX(max_peak_diff, FFABS(clock[i][0] - clock[i-1][0])); + code[cnt].size = LAG + len - last; + code[cnt].bit = hold; - if (max_peak_diff > s->max_peak_diff) - return; + return cnt + 1; +} - max = 0; min = INT_MAX; - for (i = 1; i < 7; i++) { - max = FFMAX(max, FFABS(clock[i][1] - clock[i-1][1])); - min = FFMIN(min, FFABS(clock[i][1] - clock[i-1][1])); +static void dump_code(AVFilterContext *ctx, ScanItem *scan, int len, int item) +{ + av_log(ctx, AV_LOG_DEBUG, "%d:", item); + for (int i = 0; i < len; i++) { + av_log(ctx, AV_LOG_DEBUG, " %03d", scan->code[i].size); } + av_log(ctx, AV_LOG_DEBUG, "\n"); +} + +#define READ_LINE(type, name) \ +static void read_##name(AVFrame *in, int nb_line, LineItem *line, int lp, int w) \ +{ \ + const type *src = (const type *)(&in->data[0][nb_line * in->linesize[0]]);\ + \ + if (lp) { \ + for (int i = 0; i < w; i++) { \ + int a = FFMAX(i - 3, 0); \ + int b = FFMAX(i - 2, 0); \ + int c = FFMAX(i - 1, 0); \ + int d = FFMIN(i + 3, w-1); \ + int e = FFMIN(i + 2, w-1); \ + int f = FFMIN(i + 1, w-1); \ + \ + line[LAG + i].input = (src[a] + src[b] + src[c] + src[i] + \ + src[d] + src[e] + src[f] + 6) / 7; \ + } \ + } else { \ + for (int i = 0; i < w; i++) { \ + line[LAG + i].input = src[i]; \ + } \ + } \ +} - range = max - min; - if (range > s->max_period_diff) +READ_LINE(uint8_t, byte) +READ_LINE(uint16_t, word) + +static int config_input(AVFilterLink *inlink) +{ + AVFilterContext *ctx = inlink->dst; + ReadEIA608Context *s = ctx->priv; + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); + + if (!desc) + return AVERROR_BUG; + s->depth = desc->comp[0].depth; + s->max = (1 << desc->comp[0].depth) - 1; + s->read_line[0] = read_byte; + s->read_line[1] = read_word; + + return config_filter(ctx, s->start, s->end); +} + +static void extract_line(AVFilterContext *ctx, AVFrame *in, ScanItem *scan, int w, int nb_line) +{ + ReadEIA608Context *s = ctx->priv; + LineItem *line = scan->line; + int i, j, ch, len; + uint8_t codes[19] = { 0 }; + float bit_size = 0.f; + int parity; + + memset(line, 0, (w + LAG) * sizeof(*line)); + scan->byte[0] = scan->byte[1] = 0; + scan->found = 0; + + s->read_line[s->depth > 8](in, nb_line, line, s->lp, w); + + build_histogram(s, scan, line, w); + find_black_and_white(s, scan); + if (scan->white - scan->black < 5) + return; + + thresholding(s, scan, line, LAG, 1, 0, w); + len = periods(line, scan->code, w); + dump_code(ctx, scan, len, nb_line); + if (len < 15 || + scan->code[14].bit != 0 || + w / (float)scan->code[14].size < SYNC_BITSIZE_MIN || + w / (float)scan->code[14].size > SYNC_BITSIZE_MAX) { return; + } + + for (i = 14; i < len; i++) { + bit_size += scan->code[i].size; + } - s1 = src[sync_width + width_per_bit * 0 + width_per_bit / 2]; - s2 = src[sync_width + width_per_bit * 1 + width_per_bit / 2]; - s3 = src[sync_width + width_per_bit * 2 + width_per_bit / 2]; + bit_size /= 19.f; + for (i = 1; i < 14; i++) { + if (scan->code[i].size / bit_size > CLOCK_BITSIZE_MAX || + scan->code[i].size / bit_size < CLOCK_BITSIZE_MIN) { + return; + } + } - if (FFABS(s1 - s2) > s->max_start_diff || s1 > s->black || s2 > s->black || s3 < s->white) + if (scan->code[15].size / bit_size < 0.45f) { return; + } + + for (j = 0, i = 14; i < len; i++) { + int run, bit; + + run = lrintf(scan->code[i].size / bit_size); + bit = scan->code[i].bit; + + for (int k = 0; j < 19 && k < run; k++) { + codes[j++] = bit; + } + + if (j >= 19) + break; + } for (ch = 0; ch < 2; ch++) { for (parity = 0, i = 0; i < 8; i++) { - int b = src[sync_width + width_per_bit * (i + 3 + 8 * ch) + width_per_bit / 2]; + int b = codes[3 + ch * 8 + i]; - if (b - s1 > (s3 - s1) * s->bhd) { - b = 1; + if (b == 255) { parity++; + b = 1; } else { b = 0; } - byte[ch] |= b << i; + scan->byte[ch] |= b << i; } if (s->chp) { if (!(parity & 1)) { - byte[ch] = 0; + scan->byte[ch] = 0x7F; } } } - { - uint8_t key[128], value[128]; + scan->nb_line = nb_line; + scan->found = 1; +} - snprintf(key, sizeof(key), "lavfi.readeia608.%d.cc", s->nb_found); - snprintf(value, sizeof(value), "0x%02X%02X", byte[0], byte[1]); - av_dict_set(&in->metadata, key, value, 0); +static int extract_lines(AVFilterContext *ctx, void *arg, + int job, int nb_jobs) +{ + ReadEIA608Context *s = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + const int h = s->end - s->start + 1; + const int start = (h * job) / nb_jobs; + const int end = (h * (job+1)) / nb_jobs; + AVFrame *in = arg; - snprintf(key, sizeof(key), "lavfi.readeia608.%d.line", s->nb_found); - snprintf(value, sizeof(value), "%d", line); - av_dict_set(&in->metadata, key, value, 0); + for (int i = start; i < end; i++) { + ScanItem *scan = &s->scan[i]; + + extract_line(ctx, in, scan, inlink->w, i); } - s->nb_found++; + return 0; } static int filter_frame(AVFilterLink *inlink, AVFrame *in) @@ -229,15 +472,72 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) AVFilterContext *ctx = inlink->dst; AVFilterLink *outlink = ctx->outputs[0]; ReadEIA608Context *s = ctx->priv; - int i; + int nb_found; - s->nb_found = 0; - for (i = s->start; i <= s->end; i++) - extract_line(ctx, inlink, in, i); + ctx->internal->execute(ctx, extract_lines, in, NULL, FFMIN(FFMAX(s->end - s->start + 1, 1), + ff_filter_get_nb_threads(ctx))); + + nb_found = 0; + for (int i = 0; i < s->end - s->start + 1; i++) { + ScanItem *scan = &s->scan[i]; + uint8_t key[128], value[128]; + + if (!scan->found) + continue; + + //snprintf(key, sizeof(key), "lavfi.readeia608.%d.bits", nb_found); + //snprintf(value, sizeof(value), "0b%d%d%d%d%d%d%d%d 0b%d%d%d%d%d%d%d%d", codes[3]==255,codes[4]==255,codes[5]==255,codes[6]==255,codes[7]==255,codes[8]==255,codes[9]==255,codes[10]==255,codes[11]==255,codes[12]==255,codes[13]==255,codes[14]==255,codes[15]==255,codes[16]==255,codes[17]==255,codes[18]==255); + //av_dict_set(&in->metadata, key, value, 0); + + snprintf(key, sizeof(key), "lavfi.readeia608.%d.cc", nb_found); + snprintf(value, sizeof(value), "0x%02X%02X", scan->byte[0], scan->byte[1]); + av_dict_set(&in->metadata, key, value, 0); + + snprintf(key, sizeof(key), "lavfi.readeia608.%d.line", nb_found); + snprintf(value, sizeof(value), "%d", scan->nb_line); + av_dict_set(&in->metadata, key, value, 0); + + nb_found++; + } return ff_filter_frame(outlink, in); } +static av_cold void uninit(AVFilterContext *ctx) +{ + ReadEIA608Context *s = ctx->priv; + + for (int i = 0; i < s->nb_allocated; i++) { + ScanItem *scan = &s->scan[i]; + + av_freep(&scan->histogram); + av_freep(&scan->code); + av_freep(&scan->line); + } + + s->nb_allocated = 0; + av_freep(&s->scan); +} + +static int process_command(AVFilterContext *ctx, const char *cmd, const char *args, + char *res, int res_len, int flags) +{ + ReadEIA608Context *s = ctx->priv; + int ret, start = s->start, end = s->end; + + ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags); + if (ret < 0) + return ret; + + ret = config_filter(ctx, s->start, s->end); + if (ret < 0) { + s->start = start; + s->end = end; + } + + return 0; +} + static const AVFilterPad readeia608_inputs[] = { { .name = "default", @@ -256,7 +556,7 @@ static const AVFilterPad readeia608_outputs[] = { { NULL } }; -AVFilter ff_vf_readeia608 = { +const AVFilter ff_vf_readeia608 = { .name = "readeia608", .description = NULL_IF_CONFIG_SMALL("Read EIA-608 Closed Caption codes from input video and write them to frame metadata."), .priv_size = sizeof(ReadEIA608Context), @@ -264,5 +564,7 @@ AVFilter ff_vf_readeia608 = { .query_formats = query_formats, .inputs = readeia608_inputs, .outputs = readeia608_outputs, - .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC, + .uninit = uninit, + .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC | AVFILTER_FLAG_SLICE_THREADS, + .process_command = process_command, };