git.sesse.net Git - x264/blob - tools/test_x264.py

   1 #!/usr/bin/env python
   2
   3 import operator
   4
   5 from optparse import OptionGroup
   6
   7 import sys
   8
   9 from time import time
  10
  11 from digress.cli import Dispatcher as _Dispatcher
  12 from digress.errors import ComparisonError, FailedTestError, DisabledTestError
  13 from digress.testing import depends, comparer, Fixture, Case
  14 from digress.comparers import compare_pass
  15 from digress.scm import git as x264git
  16
  17 from subprocess import Popen, PIPE, STDOUT
  18
  19 import os
  20 import re
  21 import shlex
  22 import inspect
  23
  24 from random import randrange, seed
  25 from math import ceil
  26
  27 from itertools import imap, izip
  28
  29 os.chdir(os.path.join(os.path.dirname(__file__), ".."))
  30
  31 # options
  32
  33 OPTIONS = [
  34     [ "--tune %s" % t for t in ("film", "zerolatency") ],
  35     ("", "--intra-refresh"),
  36     ("", "--no-cabac"),
  37     ("", "--interlaced"),
  38     ("", "--slice-max-size 1000"),
  39     ("", "--frame-packing 5"),
  40     [ "--preset %s" % p for p in ("ultrafast",
  41                                   "superfast",
  42                                   "veryfast",
  43                                   "faster",
  44                                   "fast",
  45                                   "medium",
  46                                   "slow",
  47                                   "slower") ]
  48 ]
  49
  50 # end options
  51
  52 def compare_yuv_output(width, height):
  53     def _compare_yuv_output(file_a, file_b):
  54         size_a = os.path.getsize(file_a)
  55         size_b = os.path.getsize(file_b)
  56
  57         if size_a != size_b:
  58             raise ComparisonError("%s is not the same size as %s" % (
  59                 file_a,
  60                 file_b
  61             ))
  62
  63         BUFFER_SIZE = 8196
  64
  65         offset = 0
  66
  67         with open(file_a) as f_a:
  68             with open(file_b) as f_b:
  69                 for chunk_a, chunk_b in izip(
  70                     imap(
  71                         lambda i: f_a.read(BUFFER_SIZE),
  72                         xrange(size_a // BUFFER_SIZE + 1)
  73                     ),
  74                     imap(
  75                         lambda i: f_b.read(BUFFER_SIZE),
  76                         xrange(size_b // BUFFER_SIZE + 1)
  77                     )
  78                 ):
  79                     chunk_size = len(chunk_a)
  80
  81                     if chunk_a != chunk_b:
  82                         for i in xrange(chunk_size):
  83                             if chunk_a[i] != chunk_b[i]:
  84                                 # calculate the macroblock, plane and frame from the offset
  85                                 offs = offset + i
  86
  87                                 y_plane_area = width * height
  88                                 u_plane_area = y_plane_area + y_plane_area * 0.25
  89                                 v_plane_area = u_plane_area + y_plane_area * 0.25
  90
  91                                 pixel = offs % v_plane_area
  92                                 frame = offs // v_plane_area
  93
  94                                 if pixel < y_plane_area:
  95                                     plane = "Y"
  96
  97                                     pixel_x = pixel % width
  98                                     pixel_y = pixel // width
  99
 100                                     macroblock = (ceil(pixel_x / 16.0), ceil(pixel_y / 16.0))
 101                                 elif pixel < u_plane_area:
 102                                     plane = "U"
 103
 104                                     pixel -= y_plane_area
 105
 106                                     pixel_x = pixel % width
 107                                     pixel_y = pixel // width
 108
 109                                     macroblock = (ceil(pixel_x / 8.0), ceil(pixel_y / 8.0))
 110                                 else:
 111                                     plane = "V"
 112
 113                                     pixel -= u_plane_area
 114
 115                                     pixel_x = pixel % width
 116                                     pixel_y = pixel // width
 117
 118                                     macroblock = (ceil(pixel_x / 8.0), ceil(pixel_y / 8.0))
 119
 120                                 macroblock = tuple([ int(x) for x in macroblock ])
 121
 122                                 raise ComparisonError("%s differs from %s at frame %d, " \
 123                                                       "macroblock %s on the %s plane (offset %d)" % (
 124                                     file_a,
 125                                     file_b,
 126                                     frame,
 127                                     macroblock,
 128                                     plane,
 129                                     offs)
 130                                 )
 131
 132                     offset += chunk_size
 133
 134     return _compare_yuv_output
 135
 136 def program_exists(program):
 137     def is_exe(fpath):
 138         return os.path.exists(fpath) and os.access(fpath, os.X_OK)
 139
 140     fpath, fname = os.path.split(program)
 141
 142     if fpath:
 143         if is_exe(program):
 144             return program
 145     else:
 146         for path in os.environ["PATH"].split(os.pathsep):
 147             exe_file = os.path.join(path, program)
 148             if is_exe(exe_file):
 149                 return exe_file
 150
 151     return None
 152
 153 class x264(Fixture):
 154     scm = x264git
 155
 156 class Compile(Case):
 157     @comparer(compare_pass)
 158     def test_configure(self):
 159         Popen([
 160             "make",
 161             "distclean"
 162         ], stdout=PIPE, stderr=STDOUT).communicate()
 163
 164         configure_proc = Popen([
 165             "./configure"
 166         ] + self.fixture.dispatcher.configure, stdout=PIPE, stderr=STDOUT)
 167
 168         output = configure_proc.communicate()[0]
 169         if configure_proc.returncode != 0:
 170             raise FailedTestError("configure failed: %s" % output.replace("\n", " "))
 171
 172     @depends("configure")
 173     @comparer(compare_pass)
 174     def test_make(self):
 175         make_proc = Popen([
 176             "make",
 177             "-j5"
 178         ], stdout=PIPE, stderr=STDOUT)
 179
 180         output = make_proc.communicate()[0]
 181         if make_proc.returncode != 0:
 182             raise FailedTestError("make failed: %s" % output.replace("\n", " "))
 183
 184 _dimension_pattern = re.compile(r"\w+ [[]info[]]: (\d+)x(\d+)[pi] \d+:\d+ @ \d+/\d+ fps [(][vc]fr[)]")
 185
 186 def _YUVOutputComparisonFactory():
 187     class YUVOutputComparison(Case):
 188         _dimension_pattern = _dimension_pattern
 189
 190         depends = [ Compile ]
 191         options = []
 192
 193         def __init__(self):
 194             for name, meth in inspect.getmembers(self):
 195                 if name[:5] == "test_" and name[5:] not in self.fixture.dispatcher.yuv_tests:
 196                     delattr(self.__class__, name)
 197
 198         def _run_x264(self):
 199             x264_proc = Popen([
 200                 "./x264",
 201                 "-o",
 202                 "%s.264" % self.fixture.dispatcher.video,
 203                 "--dump-yuv",
 204                 "x264-output.yuv"
 205             ] + self.options + [
 206                 self.fixture.dispatcher.video
 207             ], stdout=PIPE, stderr=STDOUT)
 208
 209             output = x264_proc.communicate()[0]
 210             if x264_proc.returncode != 0:
 211                 raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
 212
 213             matches = _dimension_pattern.match(output)
 214
 215             return (int(matches.group(1)), int(matches.group(2)))
 216
 217         @comparer(compare_pass)
 218         def test_jm(self):
 219             if not program_exists("ldecod"): raise DisabledTestError("jm unavailable")
 220
 221             try:
 222                 runres = self._run_x264()
 223
 224                 jm_proc = Popen([
 225                     "ldecod",
 226                     "-i",
 227                     "%s.264" % self.fixture.dispatcher.video,
 228                     "-o",
 229                     "jm-output.yuv"
 230                 ], stdout=PIPE, stderr=STDOUT)
 231
 232                 output = jm_proc.communicate()[0]
 233                 if jm_proc.returncode != 0:
 234                     raise FailedTestError("jm did not complete properly: %s" % output.replace("\n", " "))
 235
 236                 try:
 237                     compare_yuv_output(*runres)("x264-output.yuv", "jm-output.yuv")
 238                 except ComparisonError, e:
 239                     raise FailedTestError(e)
 240             finally:
 241                 try: os.remove("x264-output.yuv")
 242                 except: pass
 243
 244                 try: os.remove("%s.264" % self.fixture.dispatcher.video)
 245                 except: pass
 246
 247                 try: os.remove("jm-output.yuv")
 248                 except: pass
 249
 250                 try: os.remove("log.dec")
 251                 except: pass
 252
 253                 try: os.remove("dataDec.txt")
 254                 except: pass
 255
 256         @comparer(compare_pass)
 257         def test_ffmpeg(self):
 258             if not program_exists("ffmpeg"): raise DisabledTestError("ffmpeg unavailable")
 259             try:
 260                 runres = self._run_x264()
 261
 262                 ffmpeg_proc = Popen([
 263                     "ffmpeg",
 264                     "-vsync 0",
 265                     "-i",
 266                     "%s.264" % self.fixture.dispatcher.video,
 267                     "ffmpeg-output.yuv"
 268                 ], stdout=PIPE, stderr=STDOUT)
 269
 270                 output = ffmpeg_proc.communicate()[0]
 271                 if ffmpeg_proc.returncode != 0:
 272                     raise FailedTestError("ffmpeg did not complete properly: %s" % output.replace("\n", " "))
 273
 274                 try:
 275                     compare_yuv_output(*runres)("x264-output.yuv", "ffmpeg-output.yuv")
 276                 except ComparisonError, e:
 277                     raise FailedTestError(e)
 278             finally:
 279                 try: os.remove("x264-output.yuv")
 280                 except: pass
 281
 282                 try: os.remove("%s.264" % self.fixture.dispatcher.video)
 283                 except: pass
 284
 285                 try: os.remove("ffmpeg-output.yuv")
 286                 except: pass
 287
 288     return YUVOutputComparison
 289
 290 class Regression(Case):
 291     depends = [ Compile ]
 292
 293     _psnr_pattern = re.compile(r"x264 [[]info[]]: PSNR Mean Y:\d+[.]\d+ U:\d+[.]\d+ V:\d+[.]\d+ Avg:\d+[.]\d+ Global:(\d+[.]\d+) kb/s:\d+[.]\d+")
 294     _ssim_pattern = re.compile(r"x264 [[]info[]]: SSIM Mean Y:(\d+[.]\d+) [(]\d+[.]\d+db[)]")
 295
 296     def __init__(self):
 297         if self.fixture.dispatcher.x264:
 298             self.__class__.__name__ += " %s" % " ".join(self.fixture.dispatcher.x264)
 299
 300     def test_psnr(self):
 301         try:
 302             x264_proc = Popen([
 303                 "./x264",
 304                 "-o",
 305                 "%s.264" % self.fixture.dispatcher.video,
 306                 "--psnr"
 307             ] + self.fixture.dispatcher.x264 + [
 308                 self.fixture.dispatcher.video
 309             ], stdout=PIPE, stderr=STDOUT)
 310
 311             output = x264_proc.communicate()[0]
 312
 313             if x264_proc.returncode != 0:
 314                 raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
 315
 316             for line in output.split("\n"):
 317                 if line.startswith("x264 [info]: PSNR Mean"):
 318                     return float(self._psnr_pattern.match(line).group(1))
 319
 320             raise FailedTestError("no PSNR output caught from x264")
 321         finally:
 322             try: os.remove("%s.264" % self.fixture.dispatcher.video)
 323             except: pass
 324
 325     def test_ssim(self):
 326         try:
 327             x264_proc = Popen([
 328                 "./x264",
 329                 "-o",
 330                 "%s.264" % self.fixture.dispatcher.video,
 331                 "--ssim"
 332             ] + self.fixture.dispatcher.x264 + [
 333                 self.fixture.dispatcher.video
 334             ], stdout=PIPE, stderr=STDOUT)
 335
 336             output = x264_proc.communicate()[0]
 337
 338             if x264_proc.returncode != 0:
 339                 raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
 340
 341             for line in output.split("\n"):
 342                 if line.startswith("x264 [info]: SSIM Mean"):
 343                     return float(self._ssim_pattern.match(line).group(1))
 344
 345             raise FailedTestError("no PSNR output caught from x264")
 346         finally:
 347             try: os.remove("%s.264" % self.fixture.dispatcher.video)
 348             except: pass
 349
 350 def _generate_random_commandline():
 351     commandline = []
 352
 353     for suboptions in OPTIONS:
 354         commandline.append(suboptions[randrange(0, len(suboptions))])
 355
 356     return filter(None, reduce(operator.add, [ shlex.split(opt) for opt in commandline ]))
 357
 358 _generated = []
 359
 360 fixture = x264()
 361 fixture.register_case(Compile)
 362
 363 fixture.register_case(Regression)
 364
 365 class Dispatcher(_Dispatcher):
 366     video = "akiyo_qcif.y4m"
 367     products = 50
 368     configure = []
 369     x264 = []
 370     yuv_tests = [ "jm" ]
 371
 372     def _populate_parser(self):
 373         super(Dispatcher, self)._populate_parser()
 374
 375         # don't do a whole lot with this
 376         tcase = _YUVOutputComparisonFactory()
 377
 378         yuv_tests = [ name[5:] for name, meth in filter(lambda pair: pair[0][:5] == "test_", inspect.getmembers(tcase)) ]
 379
 380         group = OptionGroup(self.optparse, "x264 testing-specific options")
 381
 382         group.add_option(
 383             "-v",
 384             "--video",
 385             metavar="FILENAME",
 386             action="callback",
 387             dest="video",
 388             type=str,
 389             callback=lambda option, opt, value, parser: setattr(self, "video", value),
 390             help="yuv video to perform testing on (default: %s)" % self.video
 391         )
 392
 393         group.add_option(
 394             "-s",
 395             "--seed",
 396             metavar="SEED",
 397             action="callback",
 398             dest="seed",
 399             type=int,
 400             callback=lambda option, opt, value, parser: setattr(self, "seed", value),
 401             help="seed for the random number generator (default: unix timestamp)"
 402         )
 403
 404         group.add_option(
 405             "-p",
 406             "--product-tests",
 407             metavar="NUM",
 408             action="callback",
 409             dest="video",
 410             type=int,
 411             callback=lambda option, opt, value, parser: setattr(self, "products", value),
 412             help="number of cartesian products to generate for yuv comparison testing (default: %d)" % self.products
 413         )
 414
 415         group.add_option(
 416             "--configure-with",
 417             metavar="FLAGS",
 418             action="callback",
 419             dest="configure",
 420             type=str,
 421             callback=lambda option, opt, value, parser: setattr(self, "configure", shlex.split(value)),
 422             help="options to run ./configure with"
 423         )
 424
 425         group.add_option(
 426             "--yuv-tests",
 427             action="callback",
 428             dest="yuv_tests",
 429             type=str,
 430             callback=lambda option, opt, value, parser: setattr(self, "yuv_tests", [
 431                 val.strip() for val in value.split(",")
 432             ]),
 433             help="select tests to run with yuv comparisons (default: %s, available: %s)" % (
 434                 ", ".join(self.yuv_tests),
 435                 ", ".join(yuv_tests)
 436             )
 437         )
 438
 439         group.add_option(
 440             "--x264-with",
 441             metavar="FLAGS",
 442             action="callback",
 443             dest="x264",
 444             type=str,
 445             callback=lambda option, opt, value, parser: setattr(self, "x264", shlex.split(value)),
 446             help="additional options to run ./x264 with"
 447         )
 448
 449         self.optparse.add_option_group(group)
 450
 451     def pre_dispatch(self):
 452         if not hasattr(self, "seed"):
 453             self.seed = int(time())
 454
 455         print "Using seed: %d" % self.seed
 456         seed(self.seed)
 457
 458         for i in xrange(self.products):
 459             YUVOutputComparison = _YUVOutputComparisonFactory()
 460
 461             commandline = _generate_random_commandline()
 462
 463             counter = 0
 464
 465             while commandline in _generated:
 466                 counter += 1
 467                 commandline = _generate_random_commandline()
 468
 469                 if counter > 100:
 470                     print >>sys.stderr, "Maximum command-line regeneration exceeded. "  \
 471                                         "Try a different seed or specify fewer products to generate."
 472                     sys.exit(1)
 473
 474             commandline += self.x264
 475
 476             _generated.append(commandline)
 477
 478             YUVOutputComparison.options = commandline
 479             YUVOutputComparison.__name__ = ("%s %s" % (YUVOutputComparison.__name__, " ".join(commandline)))
 480
 481             fixture.register_case(YUVOutputComparison)
 482
 483 Dispatcher(fixture).dispatch()