Fix an off-by-two in the filter cutoff.
[c64tapwav] / decode.cpp
1 // Copyright Steinar H. Gunderson <sgunderson@bigfoot.com>
2 // Licensed under the GPL, v2. (See the file COPYING.)
3
4 #include <stdio.h>
5 #include <string.h>
6 #include <math.h>
7 #include <assert.h>
8 #include <limits.h>
9 #include <getopt.h>
10 #include <vector>
11 #include <algorithm>
12
13 #include "audioreader.h"
14 #include "interpolate.h"
15 #include "level.h"
16 #include "tap.h"
17 #include "filter.h"
18
19 #define BUFSIZE 4096
20 #define C64_FREQUENCY 985248
21 #define SYNC_PULSE_START 1000
22 #define SYNC_PULSE_END 20000
23 #define SYNC_PULSE_LENGTH 378.0
24 #define SYNC_TEST_TOLERANCE 1.10
25
26 // SPSA options
27 #define NUM_FILTER_COEFF 32
28 #define NUM_ITER 5000
29 #define A NUM_ITER/10  // approx
30 #define INITIAL_A 0.005 // A bit of trial and error...
31 #define INITIAL_C 0.02  // This too.
32 #define GAMMA 0.166
33 #define ALPHA 1.0
34
35 static float hysteresis_limit = 3000.0 / 32768.0;
36 static bool do_calibrate = true;
37 static bool output_cycles_plot = false;
38 static bool do_crop = false;
39 static float crop_start = 0.0f, crop_end = HUGE_VAL;
40
41 static bool use_fir_filter = false;
42 static float filter_coeff[NUM_FILTER_COEFF] = { 1.0f };  // The rest is filled with 0.
43 static bool use_rc_filter = false;
44 static float rc_filter_freq;
45 static bool output_filtered = false;
46
47 static bool quiet = false;
48 static bool do_auto_level = false;
49 static bool output_leveled = false;
50 static std::vector<float> train_snap_points;
51 static bool do_train = false;
52
53 // The minimum estimated sound level (for do_auto_level) at any given point.
54 // If you decrease this, you'll be able to amplify really silent signals
55 // by more, but you'll also increase the level of silent (ie. noise-only) segments,
56 // possibly caused misdetected pulses in these segments.
57 static float min_level = 0.05f;
58
59 // between [x,x+1]
60 double find_zerocrossing(const std::vector<float> &pcm, int x)
61 {
62         if (pcm[x] == 0) {
63                 return x;
64         }
65         if (pcm[x + 1] == 0) {
66                 return x + 1;
67         }
68
69         assert(pcm[x + 1] < 0);
70         assert(pcm[x] > 0);
71
72         double upper = x;
73         double lower = x + 1;
74         while (lower - upper > 1e-3) {
75                 double mid = 0.5f * (upper + lower);
76                 if (lanczos_interpolate(pcm, mid) > 0) {
77                         upper = mid;
78                 } else {
79                         lower = mid;
80                 }
81         }
82
83         return 0.5f * (upper + lower);
84 }
85
86 struct pulse {
87         double time;  // in seconds from start
88         double len;   // in seconds
89 };
90
91 // Calibrate on the first ~25k pulses (skip a few, just to be sure).
92 double calibrate(const std::vector<pulse> &pulses) {
93         if (pulses.size() < SYNC_PULSE_END) {
94                 fprintf(stderr, "Too few pulses, not calibrating!\n");
95                 return 1.0;
96         }
97
98         int sync_pulse_end = -1;
99         double sync_pulse_stddev = -1.0;
100
101         // Compute the standard deviation (to check for uneven speeds).
102         // If it suddenly skyrockets, we assume that sync ended earlier
103         // than we thought (it should be 25000 cycles), and that we should
104         // calibrate on fewer cycles.
105         for (int try_end : { 2000, 4000, 5000, 7500, 10000, 15000, SYNC_PULSE_END }) {
106                 double sum2 = 0.0;
107                 for (int i = SYNC_PULSE_START; i < try_end; ++i) {
108                         double cycles = pulses[i].len * C64_FREQUENCY;
109                         sum2 += (cycles - SYNC_PULSE_LENGTH) * (cycles - SYNC_PULSE_LENGTH);
110                 }
111                 double stddev = sqrt(sum2 / (try_end - SYNC_PULSE_START - 1));
112                 if (sync_pulse_end != -1 && stddev > 5.0 && stddev / sync_pulse_stddev > 1.3) {
113                         fprintf(stderr, "Stopping at %d sync pulses because standard deviation would be too big (%.2f cycles); shorter-than-usual trailer?\n",
114                                 sync_pulse_end, stddev);
115                         break;
116                 }
117                 sync_pulse_end = try_end;
118                 sync_pulse_stddev = stddev;
119         }
120         if (!quiet) {
121                 fprintf(stderr, "Sync pulse length standard deviation: %.2f cycles\n",
122                         sync_pulse_stddev);
123         }
124
125         double sum = 0.0;
126         for (int i = SYNC_PULSE_START; i < sync_pulse_end; ++i) {
127                 sum += pulses[i].len;
128         }
129         double mean_length = C64_FREQUENCY * sum / (sync_pulse_end - SYNC_PULSE_START);
130         double calibration_factor = SYNC_PULSE_LENGTH / mean_length;
131         if (!quiet) {
132                 fprintf(stderr, "Calibrated sync pulse length: %.2f -> %.2f (change %+.2f%%)\n",
133                         mean_length, SYNC_PULSE_LENGTH, 100.0 * (calibration_factor - 1.0));
134         }
135
136         // Check for pulses outside +/- 10% (sign of misdetection).
137         for (int i = SYNC_PULSE_START; i < sync_pulse_end; ++i) {
138                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
139                 if (cycles < SYNC_PULSE_LENGTH / SYNC_TEST_TOLERANCE || cycles > SYNC_PULSE_LENGTH * SYNC_TEST_TOLERANCE) {
140                         fprintf(stderr, "Sync cycle with downflank at %.6f was detected at %.0f cycles; misdetect?\n",
141                                 pulses[i].time, cycles);
142                 }
143         }
144
145         return calibration_factor;
146 }
147
148 void output_tap(const std::vector<pulse>& pulses, double calibration_factor)
149 {
150         std::vector<char> tap_data;
151         for (unsigned i = 0; i < pulses.size(); ++i) {
152                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
153                 int len = lrintf(cycles / TAP_RESOLUTION);
154                 if (i > SYNC_PULSE_END && (cycles < 100 || cycles > 800)) {
155                         fprintf(stderr, "Cycle with downflank at %.6f was detected at %.0f cycles; misdetect?\n",
156                                         pulses[i].time, cycles);
157                 }
158                 if (len <= 255) {
159                         tap_data.push_back(len);
160                 } else {
161                         int overflow_len = lrintf(cycles);
162                         tap_data.push_back(0);
163                         tap_data.push_back(overflow_len & 0xff);
164                         tap_data.push_back((overflow_len >> 8) & 0xff);
165                         tap_data.push_back(overflow_len >> 16);
166                 }
167         }
168
169         tap_header hdr;
170         memcpy(hdr.identifier, "C64-TAPE-RAW", 12);
171         hdr.version = 1;
172         hdr.reserved[0] = hdr.reserved[1] = hdr.reserved[2] = 0;
173         hdr.data_len = tap_data.size();
174
175         fwrite(&hdr, sizeof(hdr), 1, stdout);
176         fwrite(tap_data.data(), tap_data.size(), 1, stdout);
177 }
178
179 static struct option long_options[] = {
180         {"auto-level",       0,                 0, 'a' },
181         {"output-leveled",   0,                 0, 'A' },
182         {"no-calibrate",     0,                 0, 's' },
183         {"plot-cycles",      0,                 0, 'p' },
184         {"hysteresis-limit", required_argument, 0, 'l' },
185         {"filter",           required_argument, 0, 'f' },
186         {"rc-filter",        required_argument, 0, 'r' },
187         {"output-filtered",  0,                 0, 'F' },
188         {"crop",             required_argument, 0, 'c' },
189         {"quiet",            0,                 0, 'q' },
190         {"help",             0,                 0, 'h' },
191         {0,                  0,                 0, 0   }
192 };
193
194 void help()
195 {
196         fprintf(stderr, "decode [OPTIONS] AUDIO-FILE > TAP-FILE\n");
197         fprintf(stderr, "\n");
198         fprintf(stderr, "  -a, --auto-level             automatically adjust amplitude levels throughout the file\n");
199         fprintf(stderr, "  -A, --output-leveled         output leveled waveform to leveled.raw\n");
200         fprintf(stderr, "  -m, --min-level              minimum estimated sound level (0..32768) for --auto-level\n");
201         fprintf(stderr, "  -s, --no-calibrate           do not try to calibrate on sync pulse length\n");
202         fprintf(stderr, "  -p, --plot-cycles            output debugging info to cycles.plot\n");
203         fprintf(stderr, "  -l, --hysteresis-limit VAL   change amplitude threshold for ignoring pulses (0..32768)\n");
204         fprintf(stderr, "  -f, --filter C1:C2:C3:...    specify FIR filter (up to %d coefficients)\n", NUM_FILTER_COEFF);
205         fprintf(stderr, "  -r, --rc-filter FREQ         send signal through a highpass RC filter with given frequency (in Hertz)\n");
206         fprintf(stderr, "  -F, --output-filtered        output filtered waveform to filtered.raw\n");
207         fprintf(stderr, "  -c, --crop START[:END]       use only the given part of the file\n");
208         fprintf(stderr, "  -t, --train LEN1:LEN2:...    train a filter for detecting any of the given number of cycles\n");
209         fprintf(stderr, "                               (implies --no-calibrate and --quiet unless overridden)\n");
210         fprintf(stderr, "  -q, --quiet                  suppress some informational messages\n");
211         fprintf(stderr, "  -h, --help                   display this help, then exit\n");
212         exit(1);
213 }
214
215 void parse_options(int argc, char **argv)
216 {
217         for ( ;; ) {
218                 int option_index = 0;
219                 int c = getopt_long(argc, argv, "aAm:spl:f:r:Fc:t:qh", long_options, &option_index);
220                 if (c == -1)
221                         break;
222
223                 switch (c) {
224                 case 'a':
225                         do_auto_level = true;
226                         break;
227
228                 case 'A':
229                         output_leveled = true;
230                         break;
231
232                 case 'm':
233                         min_level = atof(optarg) / 32768.0;
234                         break;
235
236                 case 's':
237                         do_calibrate = false;
238                         break;
239
240                 case 'p':
241                         output_cycles_plot = true;
242                         break;
243
244                 case 'l':
245                         hysteresis_limit = atof(optarg) / 32768.0;
246                         break;
247
248                 case 'f': {
249                         const char *coeffstr = strtok(optarg, ": ");
250                         int coeff_index = 0;
251                         while (coeff_index < NUM_FILTER_COEFF && coeffstr != NULL) {
252                                 filter_coeff[coeff_index++] = atof(coeffstr);
253                                 coeffstr = strtok(NULL, ": ");
254                         }
255                         use_fir_filter = true;
256                         break;
257                 }
258
259                 case 'r':
260                         use_rc_filter = true;
261                         rc_filter_freq = atof(optarg);
262                         break;
263
264                 case 'F':
265                         output_filtered = true;
266                         break;
267
268                 case 'c': {
269                         const char *cropstr = strtok(optarg, ":");
270                         crop_start = atof(cropstr);
271                         cropstr = strtok(NULL, ":");
272                         if (cropstr == NULL) {
273                                 crop_end = HUGE_VAL;
274                         } else {
275                                 crop_end = atof(cropstr);
276                         }
277                         do_crop = true;
278                         break;
279                 }
280
281                 case 't': {
282                         const char *cyclestr = strtok(optarg, ":");
283                         while (cyclestr != NULL) {
284                                 train_snap_points.push_back(atof(cyclestr));
285                                 cyclestr = strtok(NULL, ":");
286                         }
287                         do_train = true;
288
289                         // Set reasonable defaults (can be overridden later on the command line).
290                         do_calibrate = false;
291                         quiet = true;
292                         break;
293                 }
294
295                 case 'q':
296                         quiet = true;
297                         break;
298
299                 case 'h':
300                 default:
301                         help();
302                         exit(1);
303                 }
304         }
305 }
306
307 std::vector<float> crop(const std::vector<float>& pcm, float crop_start, float crop_end, int sample_rate)
308 {
309         size_t start_sample, end_sample;
310         if (crop_start >= 0.0f) {
311                 start_sample = std::min<size_t>(lrintf(crop_start * sample_rate), pcm.size());
312         }
313         if (crop_end >= 0.0f) {
314                 end_sample = std::min<size_t>(lrintf(crop_end * sample_rate), pcm.size());
315         }
316         return std::vector<float>(pcm.begin() + start_sample, pcm.begin() + end_sample);
317 }
318
319 // TODO: Support AVX here.
320 std::vector<float> do_fir_filter(const std::vector<float>& pcm, const float* filter)
321 {
322         std::vector<float> filtered_pcm;
323         filtered_pcm.reserve(pcm.size());
324         for (unsigned i = NUM_FILTER_COEFF; i < pcm.size(); ++i) {
325                 float s = 0.0f;
326                 for (int j = 0; j < NUM_FILTER_COEFF; ++j) {
327                         s += filter[j] * pcm[i - j];
328                 }
329                 filtered_pcm.push_back(s);
330         }
331
332         if (output_filtered) {
333                 FILE *fp = fopen("filtered.raw", "wb");
334                 fwrite(filtered_pcm.data(), filtered_pcm.size() * sizeof(filtered_pcm[0]), 1, fp);
335                 fclose(fp);
336         }
337
338         return filtered_pcm;
339 }
340
341 std::vector<float> do_rc_filter(const std::vector<float>& pcm, float freq, int sample_rate)
342 {
343         std::vector<float> filtered_pcm;
344         filtered_pcm.resize(pcm.size());
345         Filter filter = Filter::hpf(2.0 * M_PI * freq / sample_rate);
346         for (unsigned i = 0; i < pcm.size(); ++i) {
347                 filtered_pcm[i] = filter.update(pcm[i]);
348         }
349
350         if (output_filtered) {
351                 FILE *fp = fopen("filtered.raw", "wb");
352                 fwrite(filtered_pcm.data(), filtered_pcm.size() * sizeof(filtered_pcm[0]), 1, fp);
353                 fclose(fp);
354         }
355
356         return filtered_pcm;
357 }
358
359 std::vector<pulse> detect_pulses(const std::vector<float> &pcm, int sample_rate)
360 {
361         std::vector<pulse> pulses;
362
363         // Find the flanks.
364         int last_bit = -1;
365         double last_downflank = -1;
366         for (unsigned i = 0; i < pcm.size(); ++i) {
367                 int bit = (pcm[i] > 0) ? 1 : 0;
368                 if (bit == 0 && last_bit == 1) {
369                         // Check if we ever go up above <hysteresis_limit> before we dip down again.
370                         bool true_pulse = false;
371                         unsigned j;
372                         int min_level_after = 32767;
373                         for (j = i; j < pcm.size(); ++j) {
374                                 min_level_after = std::min<int>(min_level_after, pcm[j]);
375                                 if (pcm[j] > 0) break;
376                                 if (pcm[j] < -hysteresis_limit) {
377                                         true_pulse = true;
378                                         break;
379                                 }
380                         }
381
382                         if (!true_pulse) {
383 #if 0
384                                 fprintf(stderr, "Ignored down-flank at %.6f seconds due to hysteresis (%d < %d).\n",
385                                         double(i) / sample_rate, -min_level_after, hysteresis_limit);
386 #endif
387                                 i = j;
388                                 continue;
389                         } 
390
391                         // down-flank!
392                         double t = find_zerocrossing(pcm, i - 1) * (1.0 / sample_rate) + crop_start;
393                         if (last_downflank > 0) {
394                                 pulse p;
395                                 p.time = t;
396                                 p.len = t - last_downflank;
397                                 pulses.push_back(p);
398                         }
399                         last_downflank = t;
400                 }
401                 last_bit = bit;
402         }
403         return pulses;
404 }
405
406 void output_cycle_plot(const std::vector<pulse> &pulses, double calibration_factor)
407 {
408         FILE *fp = fopen("cycles.plot", "w");
409         for (unsigned i = 0; i < pulses.size(); ++i) {
410                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
411                 fprintf(fp, "%f %f\n", pulses[i].time, cycles);
412         }
413         fclose(fp);
414 }
415
416 std::pair<int, double> find_closest_point(double x, const std::vector<float> &points)
417 {
418         int best_point = 0;
419         double best_dist = (x - points[0]) * (x - points[0]);
420         for (unsigned j = 1; j < train_snap_points.size(); ++j) {
421                 double dist = (x - points[j]) * (x - points[j]);
422                 if (dist < best_dist) {
423                         best_point = j;
424                         best_dist = dist;
425                 }
426         }
427         return std::make_pair(best_point, best_dist);
428 }
429
430 float eval_badness(const std::vector<pulse>& pulses, double calibration_factor)
431 {
432         double sum_badness = 0.0;
433         for (unsigned i = 0; i < pulses.size(); ++i) {
434                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
435                 if (cycles > 2000.0) cycles = 2000.0;  // Don't make pauses arbitrarily bad.
436                 std::pair<int, double> selected_point_and_sq_dist = find_closest_point(cycles, train_snap_points);
437                 sum_badness += selected_point_and_sq_dist.second;
438         }
439         return sqrt(sum_badness / (pulses.size() - 1));
440 }
441
442 void find_kmeans(const std::vector<pulse> &pulses, double calibration_factor, const std::vector<float> &initial_centers)
443 {
444         std::vector<float> last_centers = initial_centers;
445         std::vector<float> sums;
446         std::vector<float> num;
447         sums.resize(initial_centers.size());
448         num.resize(initial_centers.size());
449         for ( ;; ) {
450                 for (unsigned i = 0; i < initial_centers.size(); ++i) {
451                         sums[i] = 0.0f;
452                         num[i] = 0;
453                 }
454                 for (unsigned i = 0; i < pulses.size(); ++i) {
455                         double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
456                         // Ignore heavy outliers, which are almost always long pauses.
457                         if (cycles > 2000.0) {
458                                 continue;
459                         }
460                         std::pair<int, double> selected_point_and_sq_dist = find_closest_point(cycles, last_centers);
461                         int p = selected_point_and_sq_dist.first;
462                         sums[p] += cycles;
463                         ++num[p];
464                 }
465                 bool any_moved = false;
466                 for (unsigned i = 0; i < initial_centers.size(); ++i) {
467                         if (num[i] == 0) {
468                                 printf("K-means broke down, can't output new reference training points\n");
469                                 return;
470                         }
471                         float new_center = sums[i] / num[i];
472                         if (fabs(new_center - last_centers[i]) > 1e-3) {
473                                 any_moved = true;
474                         }
475                         last_centers[i] = new_center;
476                 }
477                 if (!any_moved) {
478                         break;
479                 }
480         }
481         printf("New reference training points:");
482         for (unsigned i = 0; i < last_centers.size(); ++i) {
483                 printf(" %.3f", last_centers[i]);
484         }
485         printf("\n");
486 }
487
488 void spsa_train(const std::vector<float> &pcm, int sample_rate)
489 {
490         float filter[NUM_FILTER_COEFF] = { 1.0f };  // The rest is filled with 0.
491
492         float start_c = INITIAL_C;
493         double best_badness = HUGE_VAL;
494
495         for (int n = 1; n < NUM_ITER; ++n) {
496                 float a = INITIAL_A * pow(n + A, -ALPHA);
497                 float c = start_c * pow(n, -GAMMA);
498
499                 // find a random perturbation
500                 float p[NUM_FILTER_COEFF];
501                 float filter1[NUM_FILTER_COEFF], filter2[NUM_FILTER_COEFF];
502                 for (int i = 0; i < NUM_FILTER_COEFF; ++i) {
503                         p[i] = (rand() % 2) ? 1.0 : -1.0;
504                         filter1[i] = std::max(std::min(filter[i] - c * p[i], 1.0f), -1.0f);
505                         filter2[i] = std::max(std::min(filter[i] + c * p[i], 1.0f), -1.0f);
506                 }
507
508                 std::vector<pulse> pulses1 = detect_pulses(do_fir_filter(pcm, filter1), sample_rate);
509                 std::vector<pulse> pulses2 = detect_pulses(do_fir_filter(pcm, filter2), sample_rate);
510                 float badness1 = eval_badness(pulses1, 1.0);
511                 float badness2 = eval_badness(pulses2, 1.0);
512
513                 // Find the gradient estimator
514                 float g[NUM_FILTER_COEFF];
515                 for (int i = 0; i < NUM_FILTER_COEFF; ++i) {
516                         g[i] = (badness2 - badness1) / (2.0 * c * p[i]);
517                         filter[i] -= a * g[i];
518                         filter[i] = std::max(std::min(filter[i], 1.0f), -1.0f);
519                 }
520                 if (badness2 < badness1) {
521                         std::swap(badness1, badness2);
522                         std::swap(filter1, filter2);
523                         std::swap(pulses1, pulses2);
524                 }
525                 if (badness1 < best_badness) {
526                         printf("\nNew best filter (badness=%f):", badness1);
527                         for (int i = 0; i < NUM_FILTER_COEFF; ++i) {
528                                 printf(" %.5f", filter1[i]);
529                         }
530                         best_badness = badness1;
531                         printf("\n");
532
533                         find_kmeans(pulses1, 1.0, train_snap_points);
534
535                         if (output_cycles_plot) {
536                                 output_cycle_plot(pulses1, 1.0);
537                         }
538                 }
539                 printf("%d ", n);
540                 fflush(stdout);
541         }
542 }
543
544 int main(int argc, char **argv)
545 {
546         parse_options(argc, argv);
547
548         make_lanczos_weight_table();
549         std::vector<float> pcm;
550         int sample_rate;
551         if (!read_audio_file(argv[optind], &pcm, &sample_rate)) {
552                 exit(1);
553         }
554
555         if (do_crop) {
556                 pcm = crop(pcm, crop_start, crop_end, sample_rate);
557         }
558
559         if (use_fir_filter) {
560                 pcm = do_fir_filter(pcm, filter_coeff);
561         }
562
563         if (use_rc_filter) {
564                 pcm = do_rc_filter(pcm, rc_filter_freq, sample_rate);
565         }
566
567         if (do_auto_level) {
568                 pcm = level_samples(pcm, min_level, sample_rate);
569                 if (output_leveled) {
570                         FILE *fp = fopen("leveled.raw", "wb");
571                         fwrite(pcm.data(), pcm.size() * sizeof(pcm[0]), 1, fp);
572                         fclose(fp);
573                 }
574         }
575
576 #if 0
577         for (int i = 0; i < LEN; ++i) {
578                 in[i] += rand() % 10000;
579         }
580 #endif
581
582 #if 0
583         for (int i = 0; i < LEN; ++i) {
584                 printf("%d\n", in[i]);
585         }
586 #endif
587
588         if (do_train) {
589                 spsa_train(pcm, sample_rate);
590                 exit(0);
591         }
592
593         std::vector<pulse> pulses = detect_pulses(pcm, sample_rate);
594
595         double calibration_factor = 1.0;
596         if (do_calibrate) {
597                 calibration_factor = calibrate(pulses);
598         }
599
600         if (output_cycles_plot) {
601                 output_cycle_plot(pulses, calibration_factor);
602         }
603
604         output_tap(pulses, calibration_factor);
605 }