3 # regression-test.pl: tests different versions of x264
4 # by Alex Izvorski & Loren Merrit 2007
21 # the following require a make testclean
22 # - changing x264 option sets and or adding/removing option sets
23 # - changing configure options
24 # - changing CFLAGS or other variables that affect compilation
25 # - a newer head revision
27 # the following do not require a make testclean, but may cause some tests to be rerun unnecessarily:
28 # - adding/removing input files
29 # - adding/removing versions
35 GetOptions ("version=s" => \@versions,
36 "input=s" => \@input_files,
37 "options=s" => \@option_sets,
41 # TODO some way to give make options
42 # TODO some way to give configure options
44 if (scalar(@versions) == 0)
46 @versions = ('rHEAD', 'current');
49 if (scalar(@option_sets) == 0 ||
50 scalar(@input_files) == 0)
52 print "Regression test for x264\n";
53 print "Usage:\n perl tools/regression-test.pl --version=623 --version=624 --input=football_720x480p.yuv --input=foreman_352x288p.yuv --options='--me=dia --subme=2 --no-cabac'\n";
54 print "Any number of versions, option sets, and input files may be given.\n";
55 print "Versions may be any svn revision, a comma-separated list of revisions, or 'current' for the version in the current directory.\n";
61 @versions = map { split m![,\s]\s*! } @versions;
63 foreach my $version (@versions)
65 $version =~ s!^head$!rHEAD!i;
66 $version =~ s!^current$!current!i;
67 $version =~ s!^(\d+)$!r$1!;
69 if (-e "test/x264-$version/x264" && ($version ne "current"))
71 print("have version: $version\n");
74 print("building version: $version\n");
75 if ($version eq "current")
77 system("(./configure && make) &> test/build.log");
78 mkpath("test/x264-$version");
79 if (! -e "x264") { print("build failed \n"); exit 1; }
80 copy("x264", "test/x264-$version/x264");
81 chmod(0755, "test/x264-$version/x264");
84 system("svn checkout -$version svn://svn.videolan.org/x264/trunk/ test/x264-$version >/dev/null");
85 chdir("test/x264-$version");
86 system("(./configure && make) &> build.log");
88 if (! -e "test/x264-$version/x264") { print("build failed \n"); exit 1; }
92 foreach my $i (0 .. scalar(@option_sets)-1)
94 $opt = $option_sets[$i];
95 print("options: $opt \n");
96 foreach my $j (0 .. scalar(@input_files)-1)
98 my $file = $input_files[$j];
99 print("input file: $file \n");
101 my $outfile = basename($file);
102 $outfile =~ s!\.yuv$!!;
103 $outfile = "test/opt$i-$outfile$j";
105 foreach my $k (0 .. scalar(@versions)-1)
107 my $version = $versions[$k];
109 if (-e "$outfile-$version.log" && ($version ne "current"))
111 print("have results for version: $version\n");
115 print("running version: $version \n");
116 # verbose option is required for frame-by-frame comparison
117 system("test/x264-$version/x264 --verbose $opt $file -o $outfile-$version.264 > $outfile-$version.log 2>&1");
120 # TODO check for crashes
121 # TODO if (read_file("$outfile-$version.log") =~ m!could not open input file!) ...
122 # TODO check decompression with jm
123 # TODO dump (and check) frames
127 my $baseversion = $versions[$k - 1];
128 print("comparing $version with $baseversion: ");
130 $is_diff ||= compare_logs("$outfile-$version.log", "$outfile-$baseversion.log");
131 $is_diff ||= compare_raw264("$outfile-$version.264", "$outfile-$baseversion.264");
132 if (! $is_diff) { print("identical \n"); }
133 $any_diff ||= $is_diff;
140 if (! $any_diff) { print "no differences found\n"; }
141 else { print "some differences found\n"; exit 1; }
146 my ($log1, $log2) = @_;
148 my $logdata1 = read_file($log1);
149 my $logdata2 = read_file($log2);
151 # FIXME comparing versions with different log output format will fail
152 $logdata1 = join("\n", grep { m!frame=! } split(m!\n!, $logdata1));
153 $logdata2 = join("\n", grep { m!frame=! } split(m!\n!, $logdata2));
156 if ($logdata1 ne $logdata2)
158 print("log files differ \n");
162 my $stats1 = parse_log_overall_stats($log1);
163 my $stats2 = parse_log_overall_stats($log2);
165 if ($stats1->{psnr_y} != $stats2->{psnr_y})
167 printf("psnr change: %+f dB \n", $stats1->{psnr_y} - $stats2->{psnr_y});
170 if ($stats1->{bitrate} != $stats2->{bitrate})
172 printf("bitrate change: %+f kb/s \n", $stats1->{bitrate} - $stats2->{bitrate});
176 #arbitrarily set cutoff to 3% change
177 #$speed_change_min = 0.03;
178 #if (abs($stats1->{fps} - $stats2->{fps}) > $speed_change_min * ($stats1->{fps} + $stats2->{fps})/2)
180 # printf("speed change: %+f fps \n", $stats1->{fps} - $stats2->{fps});
186 # TODO compare frame by frame PSNR/SSIM, record improved or unimproved ranges
187 #parse_log_frame_stats($log1);
188 #parse_log_frame_stats($log2);
190 # TODO compare actual run times
195 my ($raw1, $raw2) = @_;
197 # FIXME this may use a lot of memory
198 my $rawdata1 = read_file($raw1);
199 my $rawdata2 = read_file($raw2);
201 # remove first NAL, it is a version-specific SEI NAL
202 my $pat = chr(0).chr(0).chr(1);
203 $rawdata1 =~ s!^.*?$pat.*?$pat!$pat!;
204 $rawdata2 =~ s!^.*?$pat.*?$pat!$pat!;
206 if ($rawdata1 ne $rawdata2)
208 print("compressed files differ \n");
215 sub parse_log_frame_stats
218 my $logtext = read_file($log);
221 while ($logtext =~ m!x264 \[debug]: (frame=.*)!g)
224 if ($line !~ m!frame=\s*(\d+)\s* QP=\s*(\d+)\s* NAL=(\d+)\s* Slice:(\w)\s* Poc:(\d+)\s* I:(\d+)\s* P:(\d+)\s* SKIP:(\d+)\s* size=(\d+) bytes PSNR Y:(\d+\.\d+) U:(\d+\.\d+) V:(\d+\.\d+)!)
226 print "error: unparseable log line: $line \n"; next;
245 if ($line =~ m!SSIM Y:(\d+\.\d+)!)
250 push(@frames, $frame);
256 sub parse_log_overall_stats
259 my $logtext = read_file($log);
261 if ($logtext !~ m!x264 \[info\]: PSNR Mean Y:(\d+\.\d+) U:(\d+\.\d+) V:(\d+\.\d+) Avg:(\d+\.\d+) Global:(\d+\.\d+) kb/s:(\d+\.\d+)!)
263 print "error: unparseable log summary info \n"; return +{};
275 if ($logtext !~ m!encoded (\d+) frames, (\d+\.\d+) fps!)
277 print "error: unparseable log summary info \n"; return +{};
279 $stats->{num_frames} = $1;
288 open(F, $file) || die "could not open $file: $!";
298 my ($file, $data) = @_;
299 open(F, ">".$file) || die "could not open $file: $!";