// copes better with quantization, and calculates the
// actual whitened result as it goes.
-static void modified_levinson_durbin(int *window, int window_entries,
+static int modified_levinson_durbin(int *window, int window_entries,
int *out, int out_entries, int channels, int *tap_quant)
{
int i;
int *state = av_calloc(window_entries, sizeof(*state));
+ if (!state)
+ return AVERROR(ENOMEM);
+
memcpy(state, window, 4* window_entries);
for (i = 0; i < out_entries; i++)
}
av_free(state);
+ return 0;
}
static inline int code_samplerate(int samplerate)
// generate taps
s->tap_quant = av_calloc(s->num_taps, sizeof(*s->tap_quant));
+ if (!s->tap_quant)
+ return AVERROR(ENOMEM);
+
for (i = 0; i < s->num_taps; i++)
s->tap_quant[i] = ff_sqrt(i+1);
s->window_size = ((2*s->tail_size)+s->frame_size);
s->window = av_calloc(s->window_size, sizeof(*s->window));
- if (!s->window)
+ if (!s->window || !s->int_samples)
return AVERROR(ENOMEM);
avctx->extradata = av_mallocz(16);
s->tail[i] = s->int_samples[s->frame_size - s->tail_size + i];
// generate taps
- modified_levinson_durbin(s->window, s->window_size,
+ ret = modified_levinson_durbin(s->window, s->window_size,
s->predictor_k, s->num_taps, s->channels, s->tap_quant);
+ if (ret < 0)
+ return ret;
+
if ((ret = intlist_write(&c, state, s->predictor_k, s->num_taps, 0)) < 0)
return ret;
// generate taps
s->tap_quant = av_calloc(s->num_taps, sizeof(*s->tap_quant));
+ if (!s->tap_quant)
+ return AVERROR(ENOMEM);
+
for (i = 0; i < s->num_taps; i++)
s->tap_quant[i] = ff_sqrt(i+1);
return AVERROR(ENOMEM);
}
s->int_samples = av_calloc(s->frame_size, sizeof(*s->int_samples));
+ if (!s->int_samples)
+ return AVERROR(ENOMEM);
avctx->sample_fmt = AV_SAMPLE_FMT_S16;
return 0;