]> git.sesse.net Git - c64tapwav/blob - decode.cpp
4c5da355d2255b5565e788f2bf58716c2a81ff8e
[c64tapwav] / decode.cpp
1 // Copyright Steinar H. Gunderson <sgunderson@bigfoot.com>
2 // Licensed under the GPL, v2. (See the file COPYING.)
3
4 #include <stdio.h>
5 #include <string.h>
6 #include <math.h>
7 #include <assert.h>
8 #include <limits.h>
9 #include <getopt.h>
10 #include <vector>
11 #include <algorithm>
12
13 #include "audioreader.h"
14 #include "interpolate.h"
15 #include "level.h"
16 #include "tap.h"
17 #include "filter.h"
18
19 #define BUFSIZE 4096
20 #define C64_FREQUENCY 985248
21 #define SYNC_PULSE_START 1000
22 #define SYNC_PULSE_END 20000
23 #define SYNC_PULSE_LENGTH 378.0
24 #define SYNC_TEST_TOLERANCE 1.10
25
26 // SPSA options
27 #define NUM_FILTER_COEFF 32
28 #define NUM_ITER 5000
29 #define A NUM_ITER/10  // approx
30 #define INITIAL_A 0.005 // A bit of trial and error...
31 #define INITIAL_C 0.02  // This too.
32 #define GAMMA 0.166
33 #define ALPHA 1.0
34
35 static float hysteresis_upper_limit = 3000.0 / 32768.0;
36 static float hysteresis_lower_limit = -3000.0 / 32768.0;
37 static bool do_calibrate = true;
38 static bool output_cycles_plot = false;
39 static bool do_crop = false;
40 static float crop_start = 0.0f, crop_end = HUGE_VAL;
41
42 static bool use_fir_filter = false;
43 static float filter_coeff[NUM_FILTER_COEFF] = { 1.0f };  // The rest is filled with 0.
44 static bool use_rc_filter = false;
45 static float rc_filter_freq;
46 static bool output_filtered = false;
47
48 static bool quiet = false;
49 static bool do_auto_level = false;
50 static bool output_leveled = false;
51 static std::vector<float> train_snap_points;
52 static bool do_train = false;
53
54 // The minimum estimated sound level (for do_auto_level) at any given point.
55 // If you decrease this, you'll be able to amplify really silent signals
56 // by more, but you'll also increase the level of silent (ie. noise-only) segments,
57 // possibly caused misdetected pulses in these segments.
58 static float min_level = 0.05f;
59
60 // search for the value <limit> between [x,x+1]
61 double find_crossing(const std::vector<float> &pcm, int x, float limit)
62 {
63         double upper = x;
64         double lower = x + 1;
65         while (lower - upper > 1e-3) {
66                 double mid = 0.5f * (upper + lower);
67                 if (lanczos_interpolate(pcm, mid) > limit) {
68                         upper = mid;
69                 } else {
70                         lower = mid;
71                 }
72         }
73
74         return 0.5f * (upper + lower);
75 }
76
77 struct pulse {
78         double time;  // in seconds from start
79         double len;   // in seconds
80 };
81
82 // Calibrate on the first ~25k pulses (skip a few, just to be sure).
83 double calibrate(const std::vector<pulse> &pulses) {
84         if (pulses.size() < SYNC_PULSE_END) {
85                 fprintf(stderr, "Too few pulses, not calibrating!\n");
86                 return 1.0;
87         }
88
89         int sync_pulse_end = -1;
90         double sync_pulse_stddev = -1.0;
91
92         // Compute the standard deviation (to check for uneven speeds).
93         // If it suddenly skyrockets, we assume that sync ended earlier
94         // than we thought (it should be 25000 cycles), and that we should
95         // calibrate on fewer cycles.
96         for (int try_end : { 2000, 4000, 5000, 7500, 10000, 15000, SYNC_PULSE_END }) {
97                 double sum2 = 0.0;
98                 for (int i = SYNC_PULSE_START; i < try_end; ++i) {
99                         double cycles = pulses[i].len * C64_FREQUENCY;
100                         sum2 += (cycles - SYNC_PULSE_LENGTH) * (cycles - SYNC_PULSE_LENGTH);
101                 }
102                 double stddev = sqrt(sum2 / (try_end - SYNC_PULSE_START - 1));
103                 if (sync_pulse_end != -1 && stddev > 5.0 && stddev / sync_pulse_stddev > 1.3) {
104                         fprintf(stderr, "Stopping at %d sync pulses because standard deviation would be too big (%.2f cycles); shorter-than-usual trailer?\n",
105                                 sync_pulse_end, stddev);
106                         break;
107                 }
108                 sync_pulse_end = try_end;
109                 sync_pulse_stddev = stddev;
110         }
111         if (!quiet) {
112                 fprintf(stderr, "Sync pulse length standard deviation: %.2f cycles\n",
113                         sync_pulse_stddev);
114         }
115
116         double sum = 0.0;
117         for (int i = SYNC_PULSE_START; i < sync_pulse_end; ++i) {
118                 sum += pulses[i].len;
119         }
120         double mean_length = C64_FREQUENCY * sum / (sync_pulse_end - SYNC_PULSE_START);
121         double calibration_factor = SYNC_PULSE_LENGTH / mean_length;
122         if (!quiet) {
123                 fprintf(stderr, "Calibrated sync pulse length: %.2f -> %.2f (change %+.2f%%)\n",
124                         mean_length, SYNC_PULSE_LENGTH, 100.0 * (calibration_factor - 1.0));
125         }
126
127         // Check for pulses outside +/- 10% (sign of misdetection).
128         for (int i = SYNC_PULSE_START; i < sync_pulse_end; ++i) {
129                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
130                 if (cycles < SYNC_PULSE_LENGTH / SYNC_TEST_TOLERANCE || cycles > SYNC_PULSE_LENGTH * SYNC_TEST_TOLERANCE) {
131                         fprintf(stderr, "Sync cycle with downflank at %.6f was detected at %.0f cycles; misdetect?\n",
132                                 pulses[i].time, cycles);
133                 }
134         }
135
136         return calibration_factor;
137 }
138
139 void output_tap(const std::vector<pulse>& pulses, double calibration_factor)
140 {
141         std::vector<char> tap_data;
142         for (unsigned i = 0; i < pulses.size(); ++i) {
143                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
144                 int len = lrintf(cycles / TAP_RESOLUTION);
145                 if (i > SYNC_PULSE_END && (cycles < 100 || cycles > 800)) {
146                         fprintf(stderr, "Cycle with downflank at %.6f was detected at %.0f cycles; misdetect?\n",
147                                         pulses[i].time, cycles);
148                 }
149                 if (len <= 255) {
150                         tap_data.push_back(len);
151                 } else {
152                         int overflow_len = lrintf(cycles);
153                         tap_data.push_back(0);
154                         tap_data.push_back(overflow_len & 0xff);
155                         tap_data.push_back((overflow_len >> 8) & 0xff);
156                         tap_data.push_back(overflow_len >> 16);
157                 }
158         }
159
160         tap_header hdr;
161         memcpy(hdr.identifier, "C64-TAPE-RAW", 12);
162         hdr.version = 1;
163         hdr.reserved[0] = hdr.reserved[1] = hdr.reserved[2] = 0;
164         hdr.data_len = tap_data.size();
165
166         fwrite(&hdr, sizeof(hdr), 1, stdout);
167         fwrite(tap_data.data(), tap_data.size(), 1, stdout);
168 }
169
170 static struct option long_options[] = {
171         {"auto-level",       0,                 0, 'a' },
172         {"output-leveled",   0,                 0, 'A' },
173         {"no-calibrate",     0,                 0, 's' },
174         {"plot-cycles",      0,                 0, 'p' },
175         {"hysteresis-limit", required_argument, 0, 'l' },
176         {"filter",           required_argument, 0, 'f' },
177         {"rc-filter",        required_argument, 0, 'r' },
178         {"output-filtered",  0,                 0, 'F' },
179         {"crop",             required_argument, 0, 'c' },
180         {"quiet",            0,                 0, 'q' },
181         {"help",             0,                 0, 'h' },
182         {0,                  0,                 0, 0   }
183 };
184
185 void help()
186 {
187         fprintf(stderr, "decode [OPTIONS] AUDIO-FILE > TAP-FILE\n");
188         fprintf(stderr, "\n");
189         fprintf(stderr, "  -a, --auto-level             automatically adjust amplitude levels throughout the file\n");
190         fprintf(stderr, "  -A, --output-leveled         output leveled waveform to leveled.raw\n");
191         fprintf(stderr, "  -m, --min-level              minimum estimated sound level (0..32768) for --auto-level\n");
192         fprintf(stderr, "  -s, --no-calibrate           do not try to calibrate on sync pulse length\n");
193         fprintf(stderr, "  -p, --plot-cycles            output debugging info to cycles.plot\n");
194         fprintf(stderr, "  -l, --hysteresis-limit VAL   change amplitude threshold for ignoring pulses (0..32768)\n");
195         fprintf(stderr, "  -f, --filter C1:C2:C3:...    specify FIR filter (up to %d coefficients)\n", NUM_FILTER_COEFF);
196         fprintf(stderr, "  -r, --rc-filter FREQ         send signal through a highpass RC filter with given frequency (in Hertz)\n");
197         fprintf(stderr, "  -F, --output-filtered        output filtered waveform to filtered.raw\n");
198         fprintf(stderr, "  -c, --crop START[:END]       use only the given part of the file\n");
199         fprintf(stderr, "  -t, --train LEN1:LEN2:...    train a filter for detecting any of the given number of cycles\n");
200         fprintf(stderr, "                               (implies --no-calibrate and --quiet unless overridden)\n");
201         fprintf(stderr, "  -q, --quiet                  suppress some informational messages\n");
202         fprintf(stderr, "  -h, --help                   display this help, then exit\n");
203         exit(1);
204 }
205
206 void parse_options(int argc, char **argv)
207 {
208         for ( ;; ) {
209                 int option_index = 0;
210                 int c = getopt_long(argc, argv, "aAm:spl:f:r:Fc:t:qh", long_options, &option_index);
211                 if (c == -1)
212                         break;
213
214                 switch (c) {
215                 case 'a':
216                         do_auto_level = true;
217                         break;
218
219                 case 'A':
220                         output_leveled = true;
221                         break;
222
223                 case 'm':
224                         min_level = atof(optarg) / 32768.0;
225                         break;
226
227                 case 's':
228                         do_calibrate = false;
229                         break;
230
231                 case 'p':
232                         output_cycles_plot = true;
233                         break;
234
235                 case 'l': {
236                         const char *hyststr = strtok(optarg, ": ");
237                         hysteresis_upper_limit = atof(hyststr) / 32768.0;
238                         hyststr = strtok(NULL, ": ");
239                         if (hyststr == NULL) {
240                                 hysteresis_lower_limit = -hysteresis_upper_limit;
241                         } else {
242                                 hysteresis_lower_limit = atof(hyststr) / 32768.0;
243                         }
244                         break;
245                 }
246
247                 case 'f': {
248                         const char *coeffstr = strtok(optarg, ": ");
249                         int coeff_index = 0;
250                         while (coeff_index < NUM_FILTER_COEFF && coeffstr != NULL) {
251                                 filter_coeff[coeff_index++] = atof(coeffstr);
252                                 coeffstr = strtok(NULL, ": ");
253                         }
254                         use_fir_filter = true;
255                         break;
256                 }
257
258                 case 'r':
259                         use_rc_filter = true;
260                         rc_filter_freq = atof(optarg);
261                         break;
262
263                 case 'F':
264                         output_filtered = true;
265                         break;
266
267                 case 'c': {
268                         const char *cropstr = strtok(optarg, ":");
269                         crop_start = atof(cropstr);
270                         cropstr = strtok(NULL, ":");
271                         if (cropstr == NULL) {
272                                 crop_end = HUGE_VAL;
273                         } else {
274                                 crop_end = atof(cropstr);
275                         }
276                         do_crop = true;
277                         break;
278                 }
279
280                 case 't': {
281                         const char *cyclestr = strtok(optarg, ":");
282                         while (cyclestr != NULL) {
283                                 train_snap_points.push_back(atof(cyclestr));
284                                 cyclestr = strtok(NULL, ":");
285                         }
286                         do_train = true;
287
288                         // Set reasonable defaults (can be overridden later on the command line).
289                         do_calibrate = false;
290                         quiet = true;
291                         break;
292                 }
293
294                 case 'q':
295                         quiet = true;
296                         break;
297
298                 case 'h':
299                 default:
300                         help();
301                         exit(1);
302                 }
303         }
304 }
305
306 std::vector<float> crop(const std::vector<float>& pcm, float crop_start, float crop_end, int sample_rate)
307 {
308         size_t start_sample, end_sample;
309         if (crop_start >= 0.0f) {
310                 start_sample = std::min<size_t>(lrintf(crop_start * sample_rate), pcm.size());
311         }
312         if (crop_end >= 0.0f) {
313                 end_sample = std::min<size_t>(lrintf(crop_end * sample_rate), pcm.size());
314         }
315         return std::vector<float>(pcm.begin() + start_sample, pcm.begin() + end_sample);
316 }
317
318 // TODO: Support AVX here.
319 std::vector<float> do_fir_filter(const std::vector<float>& pcm, const float* filter)
320 {
321         std::vector<float> filtered_pcm;
322         filtered_pcm.reserve(pcm.size());
323         for (unsigned i = NUM_FILTER_COEFF; i < pcm.size(); ++i) {
324                 float s = 0.0f;
325                 for (int j = 0; j < NUM_FILTER_COEFF; ++j) {
326                         s += filter[j] * pcm[i - j];
327                 }
328                 filtered_pcm.push_back(s);
329         }
330
331         if (output_filtered) {
332                 FILE *fp = fopen("filtered.raw", "wb");
333                 fwrite(filtered_pcm.data(), filtered_pcm.size() * sizeof(filtered_pcm[0]), 1, fp);
334                 fclose(fp);
335         }
336
337         return filtered_pcm;
338 }
339
340 std::vector<float> do_rc_filter(const std::vector<float>& pcm, float freq, int sample_rate)
341 {
342         // This is only a 6 dB/oct filter, which seemingly works better
343         // than the Filter class, which is a standard biquad (12 dB/oct).
344         // The b/c calculations come from libnyquist (atone.c);
345         // I haven't checked, but I suppose they fall out of the bilinear
346         // transform of the transfer function H(s) = s/(s + w).
347         std::vector<float> filtered_pcm;
348         filtered_pcm.resize(pcm.size());
349         const float b = 2.0f - cos(2.0 * M_PI * freq / sample_rate);
350         const float c = b - sqrt(b * b - 1.0f);
351         float prev_in = 0.0f;
352         float prev_out = 0.0f;
353         for (unsigned i = 0; i < pcm.size(); ++i) {
354                 float in = pcm[i];
355                 float out = c * (prev_out + in - prev_in);
356                 filtered_pcm[i] = out;
357                 prev_in = in;
358                 prev_out = out;
359         }
360
361         if (output_filtered) {
362                 FILE *fp = fopen("filtered.raw", "wb");
363                 fwrite(filtered_pcm.data(), filtered_pcm.size() * sizeof(filtered_pcm[0]), 1, fp);
364                 fclose(fp);
365         }
366
367         return filtered_pcm;
368 }
369
370 std::vector<pulse> detect_pulses(const std::vector<float> &pcm, int sample_rate)
371 {
372         std::vector<pulse> pulses;
373
374         // Find the flanks.
375         enum State { START, ABOVE, BELOW } state = START;
376         double last_downflank = -1;
377         for (unsigned i = 0; i < pcm.size(); ++i) {
378                 if (pcm[i] > hysteresis_upper_limit) {
379                         state = ABOVE;
380                 } else if (pcm[i] < hysteresis_lower_limit) {
381                         if (state == ABOVE) {
382                                 // down-flank!
383                                 double t = find_crossing(pcm, i - 1, hysteresis_lower_limit) * (1.0 / sample_rate) + crop_start;
384                                 if (last_downflank > 0) {
385                                         pulse p;
386                                         p.time = t;
387                                         p.len = t - last_downflank;
388                                         pulses.push_back(p);
389                                 }
390                                 last_downflank = t;
391                         }
392                         state = BELOW;
393                 }
394         }
395         return pulses;
396 }
397
398 void output_cycle_plot(const std::vector<pulse> &pulses, double calibration_factor)
399 {
400         FILE *fp = fopen("cycles.plot", "w");
401         for (unsigned i = 0; i < pulses.size(); ++i) {
402                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
403                 fprintf(fp, "%f %f\n", pulses[i].time, cycles);
404         }
405         fclose(fp);
406 }
407
408 std::pair<int, double> find_closest_point(double x, const std::vector<float> &points)
409 {
410         int best_point = 0;
411         double best_dist = (x - points[0]) * (x - points[0]);
412         for (unsigned j = 1; j < train_snap_points.size(); ++j) {
413                 double dist = (x - points[j]) * (x - points[j]);
414                 if (dist < best_dist) {
415                         best_point = j;
416                         best_dist = dist;
417                 }
418         }
419         return std::make_pair(best_point, best_dist);
420 }
421
422 float eval_badness(const std::vector<pulse>& pulses, double calibration_factor)
423 {
424         double sum_badness = 0.0;
425         for (unsigned i = 0; i < pulses.size(); ++i) {
426                 double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
427                 if (cycles > 2000.0) cycles = 2000.0;  // Don't make pauses arbitrarily bad.
428                 std::pair<int, double> selected_point_and_sq_dist = find_closest_point(cycles, train_snap_points);
429                 sum_badness += selected_point_and_sq_dist.second;
430         }
431         return sqrt(sum_badness / (pulses.size() - 1));
432 }
433
434 void find_kmeans(const std::vector<pulse> &pulses, double calibration_factor, const std::vector<float> &initial_centers)
435 {
436         std::vector<float> last_centers = initial_centers;
437         std::vector<float> sums;
438         std::vector<float> num;
439         sums.resize(initial_centers.size());
440         num.resize(initial_centers.size());
441         for ( ;; ) {
442                 for (unsigned i = 0; i < initial_centers.size(); ++i) {
443                         sums[i] = 0.0f;
444                         num[i] = 0;
445                 }
446                 for (unsigned i = 0; i < pulses.size(); ++i) {
447                         double cycles = pulses[i].len * calibration_factor * C64_FREQUENCY;
448                         // Ignore heavy outliers, which are almost always long pauses.
449                         if (cycles > 2000.0) {
450                                 continue;
451                         }
452                         std::pair<int, double> selected_point_and_sq_dist = find_closest_point(cycles, last_centers);
453                         int p = selected_point_and_sq_dist.first;
454                         sums[p] += cycles;
455                         ++num[p];
456                 }
457                 bool any_moved = false;
458                 for (unsigned i = 0; i < initial_centers.size(); ++i) {
459                         if (num[i] == 0) {
460                                 fprintf(stderr, "K-means broke down, can't output new reference training points\n");
461                                 return;
462                         }
463                         float new_center = sums[i] / num[i];
464                         if (fabs(new_center - last_centers[i]) > 1e-3) {
465                                 any_moved = true;
466                         }
467                         last_centers[i] = new_center;
468                 }
469                 if (!any_moved) {
470                         break;
471                 }
472         }
473         fprintf(stderr, "New reference training points:");
474         for (unsigned i = 0; i < last_centers.size(); ++i) {
475                 fprintf(stderr, " %.3f", last_centers[i]);
476         }
477         fprintf(stderr, "\n");
478 }
479
480 void spsa_train(const std::vector<float> &pcm, int sample_rate)
481 {
482         float filter[NUM_FILTER_COEFF] = { 1.0f };  // The rest is filled with 0.
483
484         float start_c = INITIAL_C;
485         double best_badness = HUGE_VAL;
486
487         for (int n = 1; n < NUM_ITER; ++n) {
488                 float a = INITIAL_A * pow(n + A, -ALPHA);
489                 float c = start_c * pow(n, -GAMMA);
490
491                 // find a random perturbation
492                 float p[NUM_FILTER_COEFF];
493                 float filter1[NUM_FILTER_COEFF], filter2[NUM_FILTER_COEFF];
494                 for (int i = 0; i < NUM_FILTER_COEFF; ++i) {
495                         p[i] = (rand() % 2) ? 1.0 : -1.0;
496                         filter1[i] = std::max(std::min(filter[i] - c * p[i], 1.0f), -1.0f);
497                         filter2[i] = std::max(std::min(filter[i] + c * p[i], 1.0f), -1.0f);
498                 }
499
500                 std::vector<pulse> pulses1 = detect_pulses(do_fir_filter(pcm, filter1), sample_rate);
501                 std::vector<pulse> pulses2 = detect_pulses(do_fir_filter(pcm, filter2), sample_rate);
502                 float badness1 = eval_badness(pulses1, 1.0);
503                 float badness2 = eval_badness(pulses2, 1.0);
504
505                 // Find the gradient estimator
506                 float g[NUM_FILTER_COEFF];
507                 for (int i = 0; i < NUM_FILTER_COEFF; ++i) {
508                         g[i] = (badness2 - badness1) / (2.0 * c * p[i]);
509                         filter[i] -= a * g[i];
510                         filter[i] = std::max(std::min(filter[i], 1.0f), -1.0f);
511                 }
512                 if (badness2 < badness1) {
513                         std::swap(badness1, badness2);
514                         std::swap(filter1, filter2);
515                         std::swap(pulses1, pulses2);
516                 }
517                 if (badness1 < best_badness) {
518                         printf("\nNew best filter (badness=%f):", badness1);
519                         for (int i = 0; i < NUM_FILTER_COEFF; ++i) {
520                                 printf(" %.5f", filter1[i]);
521                         }
522                         best_badness = badness1;
523                         printf("\n");
524
525                         find_kmeans(pulses1, 1.0, train_snap_points);
526
527                         if (output_cycles_plot) {
528                                 output_cycle_plot(pulses1, 1.0);
529                         }
530                 }
531                 printf("%d ", n);
532                 fflush(stdout);
533         }
534 }
535
536 int main(int argc, char **argv)
537 {
538         parse_options(argc, argv);
539
540         make_lanczos_weight_table();
541         std::vector<float> pcm;
542         int sample_rate;
543         if (!read_audio_file(argv[optind], &pcm, &sample_rate)) {
544                 exit(1);
545         }
546
547         if (do_crop) {
548                 pcm = crop(pcm, crop_start, crop_end, sample_rate);
549         }
550
551         if (use_fir_filter) {
552                 pcm = do_fir_filter(pcm, filter_coeff);
553         }
554
555         if (use_rc_filter) {
556                 pcm = do_rc_filter(pcm, rc_filter_freq, sample_rate);
557         }
558
559         if (do_auto_level) {
560                 pcm = level_samples(pcm, min_level, sample_rate);
561                 if (output_leveled) {
562                         FILE *fp = fopen("leveled.raw", "wb");
563                         fwrite(pcm.data(), pcm.size() * sizeof(pcm[0]), 1, fp);
564                         fclose(fp);
565                 }
566         }
567
568 #if 0
569         for (int i = 0; i < LEN; ++i) {
570                 in[i] += rand() % 10000;
571         }
572 #endif
573
574 #if 0
575         for (int i = 0; i < LEN; ++i) {
576                 printf("%d\n", in[i]);
577         }
578 #endif
579
580         if (do_train) {
581                 spsa_train(pcm, sample_rate);
582                 exit(0);
583         }
584
585         std::vector<pulse> pulses = detect_pulses(pcm, sample_rate);
586
587         double calibration_factor = 1.0;
588         if (do_calibrate) {
589                 calibration_factor = calibrate(pulses);
590         }
591
592         if (output_cycles_plot) {
593                 output_cycle_plot(pulses, calibration_factor);
594         }
595
596         output_tap(pulses, calibration_factor);
597 }