_ Git - youtube-dl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # Author: Benjamin Johnson
   6 # License: Public domain code
   7 import htmlentitydefs
   8 import httplib
   9 import locale
  10 import math
  11 import netrc
  12 import os
  13 import os.path
  14 import re
  15 import socket
  16 import string
  17 import subprocess
  18 import sys
  19 import time
  20 import urllib
  21 import urllib2
  22
  23 # parse_qs was moved from the cgi module to the urlparse module recently.
  24 try:
  25         from urlparse import parse_qs
  26 except ImportError:
  27         from cgi import parse_qs
  28
  29 std_headers = {
  30         'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.6) Gecko/20100627 Firefox/3.6.6',
  31         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  32         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  33         'Accept-Language': 'en-us,en;q=0.5',
  34 }
  35
  36 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  37
  38 def preferredencoding():
  39         """Get preferred encoding.
  40
  41         Returns the best encoding scheme for the system, based on
  42         locale.getpreferredencoding() and some further tweaks.
  43         """
  44         def yield_preferredencoding():
  45                 try:
  46                         pref = locale.getpreferredencoding()
  47                         u'TEST'.encode(pref)
  48                 except:
  49                         pref = 'UTF-8'
  50                 while True:
  51                         yield pref
  52         return yield_preferredencoding().next()
  53
  54 def htmlentity_transform(matchobj):
  55         """Transforms an HTML entity to a Unicode character.
  56
  57         This function receives a match object and is intended to be used with
  58         the re.sub() function.
  59         """
  60         entity = matchobj.group(1)
  61
  62         # Known non-numeric HTML entity
  63         if entity in htmlentitydefs.name2codepoint:
  64                 return unichr(htmlentitydefs.name2codepoint[entity])
  65
  66         # Unicode character
  67         mobj = re.match(ur'(?u)#(x?\d+)', entity)
  68         if mobj is not None:
  69                 numstr = mobj.group(1)
  70                 if numstr.startswith(u'x'):
  71                         base = 16
  72                         numstr = u'0%s' % numstr
  73                 else:
  74                         base = 10
  75                 return unichr(long(numstr, base))
  76
  77         # Unknown entity in name, return its literal representation
  78         return (u'&%s;' % entity)
  79
  80 def sanitize_title(utitle):
  81         """Sanitizes a video title so it could be used as part of a filename."""
  82         utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
  83         return utitle.replace(unicode(os.sep), u'%')
  84
  85 def sanitize_open(filename, open_mode):
  86         """Try to open the given filename, and slightly tweak it if this fails.
  87
  88         Attempts to open the given filename. If this fails, it tries to change
  89         the filename slightly, step by step, until it's either able to open it
  90         or it fails and raises a final exception, like the standard open()
  91         function.
  92
  93         It returns the tuple (stream, definitive_file_name).
  94         """
  95         try:
  96                 if filename == u'-':
  97                         return (sys.stdout, filename)
  98                 stream = open(filename, open_mode)
  99                 return (stream, filename)
 100         except (IOError, OSError), err:
 101                 # In case of error, try to remove win32 forbidden chars
 102                 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 103
 104                 # An exception here should be caught in the caller
 105                 stream = open(filename, open_mode)
 106                 return (stream, filename)
 107
 108
 109 class DownloadError(Exception):
 110         """Download Error exception.
 111
 112         This exception may be thrown by FileDownloader objects if they are not
 113         configured to continue on errors. They will contain the appropriate
 114         error message.
 115         """
 116         pass
 117
 118 class SameFileError(Exception):
 119         """Same File exception.
 120
 121         This exception will be thrown by FileDownloader objects if they detect
 122         multiple files would have to be downloaded to the same file on disk.
 123         """
 124         pass
 125
 126 class PostProcessingError(Exception):
 127         """Post Processing exception.
 128
 129         This exception may be raised by PostProcessor's .run() method to
 130         indicate an error in the postprocessing task.
 131         """
 132         pass
 133
 134 class UnavailableFormatError(Exception):
 135         """Unavailable Format exception.
 136
 137         This exception will be thrown when a video is requested
 138         in a format that is not available for that video.
 139         """
 140         pass
 141
 142 class ContentTooShortError(Exception):
 143         """Content Too Short exception.
 144
 145         This exception may be raised by FileDownloader objects when a file they
 146         download is too small for what the server announced first, indicating
 147         the connection was probably interrupted.
 148         """
 149         # Both in bytes
 150         downloaded = None
 151         expected = None
 152
 153         def __init__(self, downloaded, expected):
 154                 self.downloaded = downloaded
 155                 self.expected = expected
 156
 157 class FileDownloader(object):
 158         """File Downloader class.
 159
 160         File downloader objects are the ones responsible of downloading the
 161         actual video file and writing it to disk if the user has requested
 162         it, among some other tasks. In most cases there should be one per
 163         program. As, given a video URL, the downloader doesn't know how to
 164         extract all the needed information, task that InfoExtractors do, it
 165         has to pass the URL to one of them.
 166
 167         For this, file downloader objects have a method that allows
 168         InfoExtractors to be registered in a given order. When it is passed
 169         a URL, the file downloader handles it to the first InfoExtractor it
 170         finds that reports being able to handle it. The InfoExtractor extracts
 171         all the information about the video or videos the URL refers to, and
 172         asks the FileDownloader to process the video information, possibly
 173         downloading the video.
 174
 175         File downloaders accept a lot of parameters. In order not to saturate
 176         the object constructor with arguments, it receives a dictionary of
 177         options instead. These options are available through the params
 178         attribute for the InfoExtractors to use. The FileDownloader also
 179         registers itself as the downloader in charge for the InfoExtractors
 180         that are added to it, so this is a "mutual registration".
 181
 182         Available options:
 183
 184         username:       Username for authentication purposes.
 185         password:       Password for authentication purposes.
 186         usenetrc:       Use netrc for authentication instead.
 187         quiet:          Do not print messages to stdout.
 188         forceurl:       Force printing final URL.
 189         forcetitle:     Force printing title.
 190         simulate:       Do not download the video files.
 191         format:         Video format code.
 192         format_limit:   Highest quality format to try.
 193         outtmpl:        Template for output names.
 194         ignoreerrors:   Do not stop on download errors.
 195         ratelimit:      Download speed limit, in bytes/sec.
 196         nooverwrites:   Prevent overwriting files.
 197         retries:        Number of times to retry for HTTP error 503
 198         continuedl:     Try to continue downloads if possible.
 199         noprogress:     Do not print the progress bar.
 200         """
 201
 202         params = None
 203         _ies = []
 204         _pps = []
 205         _download_retcode = None
 206         _num_downloads = None
 207
 208         def __init__(self, params):
 209                 """Create a FileDownloader object with the given options."""
 210                 self._ies = []
 211                 self._pps = []
 212                 self._download_retcode = 0
 213                 self._num_downloads = 0
 214                 self.params = params
 215
 216         @staticmethod
 217         def pmkdir(filename):
 218                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 219                 components = filename.split(os.sep)
 220                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 221                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 222                 for dir in aggregate:
 223                         if not os.path.exists(dir):
 224                                 os.mkdir(dir)
 225
 226         @staticmethod
 227         def format_bytes(bytes):
 228                 if bytes is None:
 229                         return 'N/A'
 230                 if type(bytes) is str:
 231                         bytes = float(bytes)
 232                 if bytes == 0.0:
 233                         exponent = 0
 234                 else:
 235                         exponent = long(math.log(bytes, 1024.0))
 236                 suffix = 'bkMGTPEZY'[exponent]
 237                 converted = float(bytes) / float(1024**exponent)
 238                 return '%.2f%s' % (converted, suffix)
 239
 240         @staticmethod
 241         def calc_percent(byte_counter, data_len):
 242                 if data_len is None:
 243                         return '---.-%'
 244                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 245
 246         @staticmethod
 247         def calc_eta(start, now, total, current):
 248                 if total is None:
 249                         return '--:--'
 250                 dif = now - start
 251                 if current == 0 or dif < 0.001: # One millisecond
 252                         return '--:--'
 253                 rate = float(current) / dif
 254                 eta = long((float(total) - float(current)) / rate)
 255                 (eta_mins, eta_secs) = divmod(eta, 60)
 256                 if eta_mins > 99:
 257                         return '--:--'
 258                 return '%02d:%02d' % (eta_mins, eta_secs)
 259
 260         @staticmethod
 261         def calc_speed(start, now, bytes):
 262                 dif = now - start
 263                 if bytes == 0 or dif < 0.001: # One millisecond
 264                         return '%10s' % '---b/s'
 265                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 266
 267         @staticmethod
 268         def best_block_size(elapsed_time, bytes):
 269                 new_min = max(bytes / 2.0, 1.0)
 270                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 271                 if elapsed_time < 0.001:
 272                         return long(new_max)
 273                 rate = bytes / elapsed_time
 274                 if rate > new_max:
 275                         return long(new_max)
 276                 if rate < new_min:
 277                         return long(new_min)
 278                 return long(rate)
 279
 280         @staticmethod
 281         def parse_bytes(bytestr):
 282                 """Parse a string indicating a byte quantity into a long integer."""
 283                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 284                 if matchobj is None:
 285                         return None
 286                 number = float(matchobj.group(1))
 287                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 288                 return long(round(number * multiplier))
 289
 290         @staticmethod
 291         def verify_url(url):
 292                 """Verify a URL is valid and data could be downloaded. Return real data URL."""
 293                 request = urllib2.Request(url, None, std_headers)
 294                 data = urllib2.urlopen(request)
 295                 data.read(1)
 296                 url = data.geturl()
 297                 data.close()
 298                 return url
 299
 300         def add_info_extractor(self, ie):
 301                 """Add an InfoExtractor object to the end of the list."""
 302                 self._ies.append(ie)
 303                 ie.set_downloader(self)
 304
 305         def add_post_processor(self, pp):
 306                 """Add a PostProcessor object to the end of the chain."""
 307                 self._pps.append(pp)
 308                 pp.set_downloader(self)
 309
 310         def to_stdout(self, message, skip_eol=False, ignore_encoding_errors=False):
 311                 """Print message to stdout if not in quiet mode."""
 312                 try:
 313                         if not self.params.get('quiet', False):
 314                                 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
 315                         sys.stdout.flush()
 316                 except (UnicodeEncodeError), err:
 317                         if not ignore_encoding_errors:
 318                                 raise
 319
 320         def to_stderr(self, message):
 321                 """Print message to stderr."""
 322                 print >>sys.stderr, message.encode(preferredencoding())
 323
 324         def fixed_template(self):
 325                 """Checks if the output template is fixed."""
 326                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 327
 328         def trouble(self, message=None):
 329                 """Determine action to take when a download problem appears.
 330
 331                 Depending on if the downloader has been configured to ignore
 332                 download errors or not, this method may throw an exception or
 333                 not when errors are found, after printing the message.
 334                 """
 335                 if message is not None:
 336                         self.to_stderr(message)
 337                 if not self.params.get('ignoreerrors', False):
 338                         raise DownloadError(message)
 339                 self._download_retcode = 1
 340
 341         def slow_down(self, start_time, byte_counter):
 342                 """Sleep if the download speed is over the rate limit."""
 343                 rate_limit = self.params.get('ratelimit', None)
 344                 if rate_limit is None or byte_counter == 0:
 345                         return
 346                 now = time.time()
 347                 elapsed = now - start_time
 348                 if elapsed <= 0.0:
 349                         return
 350                 speed = float(byte_counter) / elapsed
 351                 if speed > rate_limit:
 352                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 353
 354         def report_destination(self, filename):
 355                 """Report destination filename."""
 356                 self.to_stdout(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
 357
 358         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 359                 """Report download progress."""
 360                 if self.params.get('noprogress', False):
 361                         return
 362                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 363                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 364
 365         def report_resuming_byte(self, resume_len):
 366                 """Report attemtp to resume at given byte."""
 367                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 368
 369         def report_retry(self, count, retries):
 370                 """Report retry in case of HTTP error 503"""
 371                 self.to_stdout(u'[download] Got HTTP error 503. Retrying (attempt %d of %d)...' % (count, retries))
 372
 373         def report_file_already_downloaded(self, file_name):
 374                 """Report file has already been fully downloaded."""
 375                 try:
 376                         self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 377                 except (UnicodeEncodeError), err:
 378                         self.to_stdout(u'[download] The file has already been downloaded')
 379
 380         def report_unable_to_resume(self):
 381                 """Report it was impossible to resume download."""
 382                 self.to_stdout(u'[download] Unable to resume')
 383
 384         def report_finish(self):
 385                 """Report download finished."""
 386                 if self.params.get('noprogress', False):
 387                         self.to_stdout(u'[download] Download completed')
 388                 else:
 389                         self.to_stdout(u'')
 390
 391         def increment_downloads(self):
 392                 """Increment the ordinal that assigns a number to each file."""
 393                 self._num_downloads += 1
 394
 395         def process_info(self, info_dict):
 396                 """Process a single dictionary returned by an InfoExtractor."""
 397                 # Do nothing else if in simulate mode
 398                 if self.params.get('simulate', False):
 399                         # Verify URL if it's an HTTP one
 400                         if info_dict['url'].startswith('http'):
 401                                 try:
 402                                         self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8')
 403                                 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 404                                         raise UnavailableFormatError
 405
 406                         # Forced printings
 407                         if self.params.get('forcetitle', False):
 408                                 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 409                         if self.params.get('forceurl', False):
 410                                 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 411                         if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 412                                 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
 413                         if self.params.get('forcedescription', False) and 'description' in info_dict:
 414                                 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 415
 416                         return
 417
 418                 try:
 419                         template_dict = dict(info_dict)
 420                         template_dict['epoch'] = unicode(long(time.time()))
 421                         template_dict['ord'] = unicode('%05d' % self._num_downloads)
 422                         filename = self.params['outtmpl'] % template_dict
 423                 except (ValueError, KeyError), err:
 424                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 425                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 426                         self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
 427                         return
 428
 429                 try:
 430                         self.pmkdir(filename)
 431                 except (OSError, IOError), err:
 432                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 433                         return
 434
 435                 try:
 436                         success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
 437                 except (OSError, IOError), err:
 438                         raise UnavailableFormatError
 439                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 440                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 441                         return
 442                 except (ContentTooShortError, ), err:
 443                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 444                         return
 445
 446                 if success:
 447                         try:
 448                                 self.post_process(filename, info_dict)
 449                         except (PostProcessingError), err:
 450                                 self.trouble('ERROR: postprocessing: %s' % str(err))
 451                                 return
 452
 453         def download(self, url_list):
 454                 """Download a given list of URLs."""
 455                 if len(url_list) > 1 and self.fixed_template():
 456                         raise SameFileError(self.params['outtmpl'])
 457
 458                 for url in url_list:
 459                         suitable_found = False
 460                         for ie in self._ies:
 461                                 # Go to next InfoExtractor if not suitable
 462                                 if not ie.suitable(url):
 463                                         continue
 464
 465                                 # Suitable InfoExtractor found
 466                                 suitable_found = True
 467
 468                                 # Extract information from URL and process it
 469                                 ie.extract(url)
 470
 471                                 # Suitable InfoExtractor had been found; go to next URL
 472                                 break
 473
 474                         if not suitable_found:
 475                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 476
 477                 return self._download_retcode
 478
 479         def post_process(self, filename, ie_info):
 480                 """Run the postprocessing chain on the given file."""
 481                 info = dict(ie_info)
 482                 info['filepath'] = filename
 483                 for pp in self._pps:
 484                         info = pp.run(info)
 485                         if info is None:
 486                                 break
 487
 488         def _download_with_rtmpdump(self, filename, url, player_url):
 489                 self.report_destination(filename)
 490
 491                 # Check for rtmpdump first
 492                 try:
 493                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 494                 except (OSError, IOError):
 495                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 496                         return False
 497
 498                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 499                 # the connection was interrumpted and resuming appears to be
 500                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 501                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename]
 502                 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
 503                 while retval == 2 or retval == 1:
 504                         prevsize = os.path.getsize(filename)
 505                         self.to_stdout(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 506                         time.sleep(5.0) # This seems to be needed
 507                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 508                         cursize = os.path.getsize(filename)
 509                         if prevsize == cursize and retval == 1:
 510                                 break
 511                 if retval == 0:
 512                         self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
 513                         return True
 514                 else:
 515                         self.trouble('\nERROR: rtmpdump exited with code %d' % retval)
 516                         return False
 517
 518         def _do_download(self, filename, url, player_url):
 519                 # Attempt to download using rtmpdump
 520                 if url.startswith('rtmp'):
 521                         return self._download_with_rtmpdump(filename, url, player_url)
 522
 523                 stream = None
 524                 open_mode = 'wb'
 525                 basic_request = urllib2.Request(url, None, std_headers)
 526                 request = urllib2.Request(url, None, std_headers)
 527
 528                 # Establish possible resume length
 529                 if os.path.isfile(filename):
 530                         resume_len = os.path.getsize(filename)
 531                 else:
 532                         resume_len = 0
 533
 534                 # Request parameters in case of being able to resume
 535                 if self.params.get('continuedl', False) and resume_len != 0:
 536                         self.report_resuming_byte(resume_len)
 537                         request.add_header('Range','bytes=%d-' % resume_len)
 538                         open_mode = 'ab'
 539
 540                 count = 0
 541                 retries = self.params.get('retries', 0)
 542                 while True:
 543                         # Establish connection
 544                         try:
 545                                 data = urllib2.urlopen(request)
 546                                 break
 547                         except (urllib2.HTTPError, ), err:
 548                                 if err.code == 503:
 549                                         # Retry in case of HTTP error 503
 550                                         count += 1
 551                                         if count <= retries:
 552                                                 self.report_retry(count, retries)
 553                                                 continue
 554                                 if err.code != 416: #  416 is 'Requested range not satisfiable'
 555                                         raise
 556                                 # Unable to resume
 557                                 data = urllib2.urlopen(basic_request)
 558                                 content_length = data.info()['Content-Length']
 559
 560                                 if content_length is not None and long(content_length) == resume_len:
 561                                         # Because the file had already been fully downloaded
 562                                         self.report_file_already_downloaded(filename)
 563                                         return True
 564                                 else:
 565                                         # Because the server didn't let us
 566                                         self.report_unable_to_resume()
 567                                         open_mode = 'wb'
 568
 569                 data_len = data.info().get('Content-length', None)
 570                 data_len_str = self.format_bytes(data_len)
 571                 byte_counter = 0
 572                 block_size = 1024
 573                 start = time.time()
 574                 while True:
 575                         # Download and write
 576                         before = time.time()
 577                         data_block = data.read(block_size)
 578                         after = time.time()
 579                         data_block_len = len(data_block)
 580                         if data_block_len == 0:
 581                                 break
 582                         byte_counter += data_block_len
 583
 584                         # Open file just in time
 585                         if stream is None:
 586                                 try:
 587                                         (stream, filename) = sanitize_open(filename, open_mode)
 588                                         self.report_destination(filename)
 589                                 except (OSError, IOError), err:
 590                                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 591                                         return False
 592                         try:
 593                                 stream.write(data_block)
 594                         except (IOError, OSError), err:
 595                                 self.trouble('\nERROR: unable to write data: %s' % str(err))
 596                         block_size = self.best_block_size(after - before, data_block_len)
 597
 598                         # Progress message
 599                         percent_str = self.calc_percent(byte_counter, data_len)
 600                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 601                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 602                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 603
 604                         # Apply rate limit
 605                         self.slow_down(start, byte_counter)
 606
 607                 self.report_finish()
 608                 if data_len is not None and str(byte_counter) != data_len:
 609                         raise ContentTooShortError(byte_counter, long(data_len))
 610                 return True
 611
 612 class InfoExtractor(object):
 613         """Information Extractor class.
 614
 615         Information extractors are the classes that, given a URL, extract
 616         information from the video (or videos) the URL refers to. This
 617         information includes the real video URL, the video title and simplified
 618         title, author and others. The information is stored in a dictionary
 619         which is then passed to the FileDownloader. The FileDownloader
 620         processes this information possibly downloading the video to the file
 621         system, among other possible outcomes. The dictionaries must include
 622         the following fields:
 623
 624         id:             Video identifier.
 625         url:            Final video URL.
 626         uploader:       Nickname of the video uploader.
 627         title:          Literal title.
 628         stitle:         Simplified title.
 629         ext:            Video filename extension.
 630         format:         Video format.
 631         player_url:     SWF Player URL (may be None).
 632
 633         The following fields are optional. Their primary purpose is to allow
 634         youtube-dl to serve as the backend for a video search function, such
 635         as the one in youtube2mp3.  They are only used when their respective
 636         forced printing functions are called:
 637
 638         thumbnail:      Full URL to a video thumbnail image.
 639         description:    One-line video description.
 640
 641         Subclasses of this one should re-define the _real_initialize() and
 642         _real_extract() methods, as well as the suitable() static method.
 643         Probably, they should also be instantiated and added to the main
 644         downloader.
 645         """
 646
 647         _ready = False
 648         _downloader = None
 649
 650         def __init__(self, downloader=None):
 651                 """Constructor. Receives an optional downloader."""
 652                 self._ready = False
 653                 self.set_downloader(downloader)
 654
 655         @staticmethod
 656         def suitable(url):
 657                 """Receives a URL and returns True if suitable for this IE."""
 658                 return False
 659
 660         def initialize(self):
 661                 """Initializes an instance (authentication, etc)."""
 662                 if not self._ready:
 663                         self._real_initialize()
 664                         self._ready = True
 665
 666         def extract(self, url):
 667                 """Extracts URL information and returns it in list of dicts."""
 668                 self.initialize()
 669                 return self._real_extract(url)
 670
 671         def set_downloader(self, downloader):
 672                 """Sets the downloader for this IE."""
 673                 self._downloader = downloader
 674
 675         def _real_initialize(self):
 676                 """Real initialization process. Redefine in subclasses."""
 677                 pass
 678
 679         def _real_extract(self, url):
 680                 """Real extraction process. Redefine in subclasses."""
 681                 pass
 682
 683 class YoutubeIE(InfoExtractor):
 684         """Information extractor for youtube.com."""
 685
 686         _VALID_URL = r'^((?:http://)?(?:youtu\.be/|(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?[\?#](?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
 687         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 688         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 689         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 690         _NETRC_MACHINE = 'youtube'
 691         # Listed in order of priority for the -b option
 692         _available_formats = ['38', '37', '22', '45', '35', '34', '43', '18', '6', '5', '17', '13', None]
 693         _video_extensions = {
 694                 '13': '3gp',
 695                 '17': 'mp4',
 696                 '18': 'mp4',
 697                 '22': 'mp4',
 698                 '37': 'mp4',
 699                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 700                 '43': 'webm',
 701                 '45': 'webm',
 702         }
 703
 704         @staticmethod
 705         def suitable(url):
 706                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 707
 708         def report_lang(self):
 709                 """Report attempt to set language."""
 710                 self._downloader.to_stdout(u'[youtube] Setting language')
 711
 712         def report_login(self):
 713                 """Report attempt to log in."""
 714                 self._downloader.to_stdout(u'[youtube] Logging in')
 715
 716         def report_age_confirmation(self):
 717                 """Report attempt to confirm age."""
 718                 self._downloader.to_stdout(u'[youtube] Confirming age')
 719
 720         def report_video_webpage_download(self, video_id):
 721                 """Report attempt to download video webpage."""
 722                 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
 723
 724         def report_video_info_webpage_download(self, video_id):
 725                 """Report attempt to download video info webpage."""
 726                 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
 727
 728         def report_information_extraction(self, video_id):
 729                 """Report attempt to extract video information."""
 730                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 731
 732         def report_unavailable_format(self, video_id, format):
 733                 """Report extracted video URL."""
 734                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 735
 736         def report_rtmp_download(self):
 737                 """Indicate the download will use the RTMP protocol."""
 738                 self._downloader.to_stdout(u'[youtube] RTMP download detected')
 739
 740         def _real_initialize(self):
 741                 if self._downloader is None:
 742                         return
 743
 744                 username = None
 745                 password = None
 746                 downloader_params = self._downloader.params
 747
 748                 # Attempt to use provided username and password or .netrc data
 749                 if downloader_params.get('username', None) is not None:
 750                         username = downloader_params['username']
 751                         password = downloader_params['password']
 752                 elif downloader_params.get('usenetrc', False):
 753                         try:
 754                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 755                                 if info is not None:
 756                                         username = info[0]
 757                                         password = info[2]
 758                                 else:
 759                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 760                         except (IOError, netrc.NetrcParseError), err:
 761                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 762                                 return
 763
 764                 # Set language
 765                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 766                 try:
 767                         self.report_lang()
 768                         urllib2.urlopen(request).read()
 769                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 770                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 771                         return
 772
 773                 # No authentication to be performed
 774                 if username is None:
 775                         return
 776
 777                 # Log in
 778                 login_form = {
 779                                 'current_form': 'loginForm',
 780                                 'next':         '/',
 781                                 'action_login': 'Log In',
 782                                 'username':     username,
 783                                 'password':     password,
 784                                 }
 785                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 786                 try:
 787                         self.report_login()
 788                         login_results = urllib2.urlopen(request).read()
 789                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 790                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 791                                 return
 792                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 793                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 794                         return
 795
 796                 # Confirm age
 797                 age_form = {
 798                                 'next_url':             '/',
 799                                 'action_confirm':       'Confirm',
 800                                 }
 801                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 802                 try:
 803                         self.report_age_confirmation()
 804                         age_results = urllib2.urlopen(request).read()
 805                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 806                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 807                         return
 808
 809         def _real_extract(self, url):
 810                 # Extract video id from URL
 811                 mobj = re.match(self._VALID_URL, url)
 812                 if mobj is None:
 813                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 814                         return
 815
 816                 # At this point we have a new video
 817                 if self._downloader is not None:
 818                         self._downloader.increment_downloads()
 819                 video_id = mobj.group(2)
 820
 821                 # Downloader parameters
 822                 best_quality = False
 823                 all_formats = False
 824                 format_param = None
 825                 quality_index = 0
 826                 if self._downloader is not None:
 827                         params = self._downloader.params
 828                         format_param = params.get('format', None)
 829                         if format_param == '0':
 830                                 format_limit = params.get('format_limit', None)
 831                                 if format_limit is not None:
 832                                         try:
 833                                                 # Start at a different format if the user has limited the maximum quality
 834                                                 quality_index = self._available_formats.index(format_limit)
 835                                         except ValueError:
 836                                                 pass
 837                                 format_param = self._available_formats[quality_index]
 838                                 best_quality = True
 839                         elif format_param == '-1':
 840                                 format_param = self._available_formats[quality_index]
 841                                 all_formats = True
 842
 843                 while True:
 844                         # Extension
 845                         video_extension = self._video_extensions.get(format_param, 'flv')
 846
 847                         # Get video webpage
 848                         self.report_video_webpage_download(video_id)
 849                         request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id, None, std_headers)
 850                         try:
 851                                 video_webpage = urllib2.urlopen(request).read()
 852                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 853                                 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 854                                 return
 855
 856                         # Attempt to extract SWF player URL
 857                         mobj = re.search(r'swfConfig.*"(http://.*?watch.*?-.*?\.swf)"', video_webpage)
 858                         if mobj is not None:
 859                                 player_url = mobj.group(1)
 860                         else:
 861                                 player_url = None
 862
 863                         # Get video info
 864                         self.report_video_info_webpage_download(video_id)
 865                         for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
 866                                 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
 867                                                    % (video_id, el_type))
 868                                 request = urllib2.Request(video_info_url, None, std_headers)
 869                                 try:
 870                                         video_info_webpage = urllib2.urlopen(request).read()
 871                                         video_info = parse_qs(video_info_webpage)
 872                                         if 'token' in video_info:
 873                                                 break
 874                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 875                                         self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 876                                         return
 877                         self.report_information_extraction(video_id)
 878
 879                         # "t" param
 880                         if 'token' not in video_info:
 881                                 # Attempt to see if YouTube has issued an error message
 882                                 if 'reason' not in video_info:
 883                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
 884                                         stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
 885                                         stream.write(video_info_webpage)
 886                                         stream.close()
 887                                 else:
 888                                         reason = urllib.unquote_plus(video_info['reason'][0])
 889                                         self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
 890                                 return
 891                         token = urllib.unquote_plus(video_info['token'][0])
 892                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
 893                         if format_param is not None:
 894                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 895
 896                         # Check possible RTMP download
 897                         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 898                                 self.report_rtmp_download()
 899                                 video_real_url = video_info['conn'][0]
 900
 901                         # uploader
 902                         if 'author' not in video_info:
 903                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 904                                 return
 905                         video_uploader = urllib.unquote_plus(video_info['author'][0])
 906
 907                         # title
 908                         if 'title' not in video_info:
 909                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 910                                 return
 911                         video_title = urllib.unquote_plus(video_info['title'][0])
 912                         video_title = video_title.decode('utf-8')
 913                         video_title = sanitize_title(video_title)
 914
 915                         # simplified title
 916                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 917                         simple_title = simple_title.strip(ur'_')
 918
 919                         # thumbnail image
 920                         if 'thumbnail_url' not in video_info:
 921                                 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
 922                                 video_thumbnail = ''
 923                         else:   # don't panic if we can't find it
 924                                 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
 925
 926                         # description
 927                         video_description = 'No description available.'
 928                         if self._downloader.params.get('forcedescription', False):
 929                                 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
 930                                 if mobj is not None:
 931                                         video_description = mobj.group(1)
 932
 933                         try:
 934                                 # Process video information
 935                                 self._downloader.process_info({
 936                                         'id':           video_id.decode('utf-8'),
 937                                         'url':          video_real_url.decode('utf-8'),
 938                                         'uploader':     video_uploader.decode('utf-8'),
 939                                         'title':        video_title,
 940                                         'stitle':       simple_title,
 941                                         'ext':          video_extension.decode('utf-8'),
 942                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
 943                                         'thumbnail':    video_thumbnail.decode('utf-8'),
 944                                         'description':  video_description.decode('utf-8'),
 945                                         'player_url':   player_url,
 946                                 })
 947
 948                                 if all_formats:
 949                                         quality_index += 1
 950                                         if quality_index == len(self._available_formats):
 951                                                 # None left to get
 952                                                 return
 953                                         else:
 954                                                 format_param = self._available_formats[quality_index]
 955                                                 continue
 956                                 return
 957
 958                         except UnavailableFormatError, err:
 959                                 if best_quality or all_formats:
 960                                         quality_index += 1
 961                                         if quality_index == len(self._available_formats):
 962                                                 # I don't ever expect this to happen
 963                                                 if not all_formats:
 964                                                         self._downloader.trouble(u'ERROR: no known formats available for video')
 965                                                 return
 966                                         else:
 967                                                 self.report_unavailable_format(video_id, format_param)
 968                                                 format_param = self._available_formats[quality_index]
 969                                                 continue
 970                                 else:
 971                                         self._downloader.trouble('ERROR: format not available for video')
 972                                         return
 973
 974
 975 class MetacafeIE(InfoExtractor):
 976         """Information Extractor for metacafe.com."""
 977
 978         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 979         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 980         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 981         _youtube_ie = None
 982
 983         def __init__(self, youtube_ie, downloader=None):
 984                 InfoExtractor.__init__(self, downloader)
 985                 self._youtube_ie = youtube_ie
 986
 987         @staticmethod
 988         def suitable(url):
 989                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 990
 991         def report_disclaimer(self):
 992                 """Report disclaimer retrieval."""
 993                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 994
 995         def report_age_confirmation(self):
 996                 """Report attempt to confirm age."""
 997                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 998
 999         def report_download_webpage(self, video_id):
1000                 """Report webpage download."""
1001                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
1002
1003         def report_extraction(self, video_id):
1004                 """Report information extraction."""
1005                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
1006
1007         def _real_initialize(self):
1008                 # Retrieve disclaimer
1009                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
1010                 try:
1011                         self.report_disclaimer()
1012                         disclaimer = urllib2.urlopen(request).read()
1013                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1014                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1015                         return
1016
1017                 # Confirm age
1018                 disclaimer_form = {
1019                         'filters': '0',
1020                         'submit': "Continue - I'm over 18",
1021                         }
1022                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
1023                 try:
1024                         self.report_age_confirmation()
1025                         disclaimer = urllib2.urlopen(request).read()
1026                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1027                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1028                         return
1029
1030         def _real_extract(self, url):
1031                 # Extract id and simplified title from URL
1032                 mobj = re.match(self._VALID_URL, url)
1033                 if mobj is None:
1034                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1035                         return
1036
1037                 video_id = mobj.group(1)
1038
1039                 # Check if video comes from YouTube
1040                 mobj2 = re.match(r'^yt-(.*)$', video_id)
1041                 if mobj2 is not None:
1042                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1043                         return
1044
1045                 # At this point we have a new video
1046                 if self._downloader is not None:
1047                         self._downloader.increment_downloads()
1048
1049                 simple_title = mobj.group(2).decode('utf-8')
1050                 video_extension = 'flv'
1051
1052                 # Retrieve video webpage to extract further information
1053                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1054                 try:
1055                         self.report_download_webpage(video_id)
1056                         webpage = urllib2.urlopen(request).read()
1057                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1058                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1059                         return
1060
1061                 # Extract URL, uploader and title from webpage
1062                 self.report_extraction(video_id)
1063                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1064                 if mobj is None:
1065                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1066                         return
1067                 mediaURL = urllib.unquote(mobj.group(1))
1068
1069                 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1070                 #if mobj is None:
1071                 #       self._downloader.trouble(u'ERROR: unable to extract gdaKey')
1072                 #       return
1073                 #gdaKey = mobj.group(1)
1074                 #
1075                 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1076
1077                 video_url = mediaURL
1078
1079                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1080                 if mobj is None:
1081                         self._downloader.trouble(u'ERROR: unable to extract title')
1082                         return
1083                 video_title = mobj.group(1).decode('utf-8')
1084                 video_title = sanitize_title(video_title)
1085
1086                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1087                 if mobj is None:
1088                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1089                         return
1090                 video_uploader = mobj.group(1)
1091
1092                 try:
1093                         # Process video information
1094                         self._downloader.process_info({
1095                                 'id':           video_id.decode('utf-8'),
1096                                 'url':          video_url.decode('utf-8'),
1097                                 'uploader':     video_uploader.decode('utf-8'),
1098                                 'title':        video_title,
1099                                 'stitle':       simple_title,
1100                                 'ext':          video_extension.decode('utf-8'),
1101                                 'format':       u'NA',
1102                                 'player_url':   None,
1103                         })
1104                 except UnavailableFormatError:
1105                         self._downloader.trouble(u'ERROR: format not available for video')
1106
1107
1108 class DailymotionIE(InfoExtractor):
1109         """Information Extractor for Dailymotion"""
1110
1111         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1112
1113         def __init__(self, downloader=None):
1114                 InfoExtractor.__init__(self, downloader)
1115
1116         @staticmethod
1117         def suitable(url):
1118                 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1119
1120         def report_download_webpage(self, video_id):
1121                 """Report webpage download."""
1122                 self._downloader.to_stdout(u'[dailymotion] %s: Downloading webpage' % video_id)
1123
1124         def report_extraction(self, video_id):
1125                 """Report information extraction."""
1126                 self._downloader.to_stdout(u'[dailymotion] %s: Extracting information' % video_id)
1127
1128         def _real_initialize(self):
1129                 return
1130
1131         def _real_extract(self, url):
1132                 # Extract id and simplified title from URL
1133                 mobj = re.match(self._VALID_URL, url)
1134                 if mobj is None:
1135                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1136                         return
1137
1138                 # At this point we have a new video
1139                 if self._downloader is not None:
1140                         self._downloader.increment_downloads()
1141                 video_id = mobj.group(1)
1142
1143                 simple_title = mobj.group(2).decode('utf-8')
1144                 video_extension = 'flv'
1145
1146                 # Retrieve video webpage to extract further information
1147                 request = urllib2.Request(url)
1148                 try:
1149                         self.report_download_webpage(video_id)
1150                         webpage = urllib2.urlopen(request).read()
1151                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1152                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1153                         return
1154
1155                 # Extract URL, uploader and title from webpage
1156                 self.report_extraction(video_id)
1157                 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1158                 if mobj is None:
1159                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1160                         return
1161                 mediaURL = urllib.unquote(mobj.group(1))
1162
1163                 # if needed add http://www.dailymotion.com/ if relative URL
1164
1165                 video_url = mediaURL
1166
1167                 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1168                 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1169                 if mobj is None:
1170                         self._downloader.trouble(u'ERROR: unable to extract title')
1171                         return
1172                 video_title = mobj.group(1).decode('utf-8')
1173                 video_title = sanitize_title(video_title)
1174
1175                 mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a></div>', webpage)
1176                 if mobj is None:
1177                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1178                         return
1179                 video_uploader = mobj.group(1)
1180
1181                 try:
1182                         # Process video information
1183                         self._downloader.process_info({
1184                                 'id':           video_id.decode('utf-8'),
1185                                 'url':          video_url.decode('utf-8'),
1186                                 'uploader':     video_uploader.decode('utf-8'),
1187                                 'title':        video_title,
1188                                 'stitle':       simple_title,
1189                                 'ext':          video_extension.decode('utf-8'),
1190                                 'format':       u'NA',
1191                                 'player_url':   None,
1192                         })
1193                 except UnavailableFormatError:
1194                         self._downloader.trouble(u'ERROR: format not available for video')
1195
1196 class GoogleIE(InfoExtractor):
1197         """Information extractor for video.google.com."""
1198
1199         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1200
1201         def __init__(self, downloader=None):
1202                 InfoExtractor.__init__(self, downloader)
1203
1204         @staticmethod
1205         def suitable(url):
1206                 return (re.match(GoogleIE._VALID_URL, url) is not None)
1207
1208         def report_download_webpage(self, video_id):
1209                 """Report webpage download."""
1210                 self._downloader.to_stdout(u'[video.google] %s: Downloading webpage' % video_id)
1211
1212         def report_extraction(self, video_id):
1213                 """Report information extraction."""
1214                 self._downloader.to_stdout(u'[video.google] %s: Extracting information' % video_id)
1215
1216         def _real_initialize(self):
1217                 return
1218
1219         def _real_extract(self, url):
1220                 # Extract id from URL
1221                 mobj = re.match(self._VALID_URL, url)
1222                 if mobj is None:
1223                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1224                         return
1225
1226                 # At this point we have a new video
1227                 if self._downloader is not None:
1228                         self._downloader.increment_downloads()
1229                 video_id = mobj.group(1)
1230
1231                 video_extension = 'mp4'
1232
1233                 # Retrieve video webpage to extract further information
1234                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1235                 try:
1236                         self.report_download_webpage(video_id)
1237                         webpage = urllib2.urlopen(request).read()
1238                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1239                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1240                         return
1241
1242                 # Extract URL, uploader, and title from webpage
1243                 self.report_extraction(video_id)
1244                 mobj = re.search(r"download_url:'([^']+)'", webpage)
1245                 if mobj is None:
1246                         video_extension = 'flv'
1247                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1248                 if mobj is None:
1249                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1250                         return
1251                 mediaURL = urllib.unquote(mobj.group(1))
1252                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1253                 mediaURL = mediaURL.replace('\\x26', '\x26')
1254
1255                 video_url = mediaURL
1256
1257                 mobj = re.search(r'<title>(.*)</title>', webpage)
1258                 if mobj is None:
1259                         self._downloader.trouble(u'ERROR: unable to extract title')
1260                         return
1261                 video_title = mobj.group(1).decode('utf-8')
1262                 video_title = sanitize_title(video_title)
1263                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1264
1265                 # Extract video description
1266                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1267                 if mobj is None:
1268                         self._downloader.trouble(u'ERROR: unable to extract video description')
1269                         return
1270                 video_description = mobj.group(1).decode('utf-8')
1271                 if not video_description:
1272                         video_description = 'No description available.'
1273
1274                 # Extract video thumbnail
1275                 if self._downloader.params.get('forcethumbnail', False):
1276                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1277                         try:
1278                                 webpage = urllib2.urlopen(request).read()
1279                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1280                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1281                                 return
1282                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1283                         if mobj is None:
1284                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1285                                 return
1286                         video_thumbnail = mobj.group(1)
1287                 else:   # we need something to pass to process_info
1288                         video_thumbnail = ''
1289
1290
1291                 try:
1292                         # Process video information
1293                         self._downloader.process_info({
1294                                 'id':           video_id.decode('utf-8'),
1295                                 'url':          video_url.decode('utf-8'),
1296                                 'uploader':     u'NA',
1297                                 'title':        video_title,
1298                                 'stitle':       simple_title,
1299                                 'ext':          video_extension.decode('utf-8'),
1300                                 'format':       u'NA',
1301                                 'player_url':   None,
1302                         })
1303                 except UnavailableFormatError:
1304                         self._downloader.trouble(u'ERROR: format not available for video')
1305
1306
1307 class PhotobucketIE(InfoExtractor):
1308         """Information extractor for photobucket.com."""
1309
1310         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1311
1312         def __init__(self, downloader=None):
1313                 InfoExtractor.__init__(self, downloader)
1314
1315         @staticmethod
1316         def suitable(url):
1317                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1318
1319         def report_download_webpage(self, video_id):
1320                 """Report webpage download."""
1321                 self._downloader.to_stdout(u'[photobucket] %s: Downloading webpage' % video_id)
1322
1323         def report_extraction(self, video_id):
1324                 """Report information extraction."""
1325                 self._downloader.to_stdout(u'[photobucket] %s: Extracting information' % video_id)
1326
1327         def _real_initialize(self):
1328                 return
1329
1330         def _real_extract(self, url):
1331                 # Extract id from URL
1332                 mobj = re.match(self._VALID_URL, url)
1333                 if mobj is None:
1334                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1335                         return
1336
1337                 # At this point we have a new video
1338                 if self._downloader is not None:
1339                         self._downloader.increment_downloads()
1340                 video_id = mobj.group(1)
1341
1342                 video_extension = 'flv'
1343
1344                 # Retrieve video webpage to extract further information
1345                 request = urllib2.Request(url)
1346                 try:
1347                         self.report_download_webpage(video_id)
1348                         webpage = urllib2.urlopen(request).read()
1349                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1350                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1351                         return
1352
1353                 # Extract URL, uploader, and title from webpage
1354                 self.report_extraction(video_id)
1355                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1356                 if mobj is None:
1357                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1358                         return
1359                 mediaURL = urllib.unquote(mobj.group(1))
1360
1361                 video_url = mediaURL
1362
1363                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1364                 if mobj is None:
1365                         self._downloader.trouble(u'ERROR: unable to extract title')
1366                         return
1367                 video_title = mobj.group(1).decode('utf-8')
1368                 video_title = sanitize_title(video_title)
1369                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1370
1371                 video_uploader = mobj.group(2).decode('utf-8')
1372
1373                 try:
1374                         # Process video information
1375                         self._downloader.process_info({
1376                                 'id':           video_id.decode('utf-8'),
1377                                 'url':          video_url.decode('utf-8'),
1378                                 'uploader':     video_uploader,
1379                                 'title':        video_title,
1380                                 'stitle':       simple_title,
1381                                 'ext':          video_extension.decode('utf-8'),
1382                                 'format':       u'NA',
1383                                 'player_url':   None,
1384                         })
1385                 except UnavailableFormatError:
1386                         self._downloader.trouble(u'ERROR: format not available for video')
1387
1388
1389 class YahooIE(InfoExtractor):
1390         """Information extractor for video.yahoo.com."""
1391
1392         # _VALID_URL matches all Yahoo! Video URLs
1393         # _VPAGE_URL matches only the extractable '/watch/' URLs
1394         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1395         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1396
1397         def __init__(self, downloader=None):
1398                 InfoExtractor.__init__(self, downloader)
1399
1400         @staticmethod
1401         def suitable(url):
1402                 return (re.match(YahooIE._VALID_URL, url) is not None)
1403
1404         def report_download_webpage(self, video_id):
1405                 """Report webpage download."""
1406                 self._downloader.to_stdout(u'[video.yahoo] %s: Downloading webpage' % video_id)
1407
1408         def report_extraction(self, video_id):
1409                 """Report information extraction."""
1410                 self._downloader.to_stdout(u'[video.yahoo] %s: Extracting information' % video_id)
1411
1412         def _real_initialize(self):
1413                 return
1414
1415         def _real_extract(self, url, new_video=True):
1416                 # Extract ID from URL
1417                 mobj = re.match(self._VALID_URL, url)
1418                 if mobj is None:
1419                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1420                         return
1421
1422                 # At this point we have a new video
1423                 if self._downloader is not None and new_video:
1424                         self._downloader.increment_downloads()
1425                 video_id = mobj.group(2)
1426                 video_extension = 'flv'
1427
1428                 # Rewrite valid but non-extractable URLs as
1429                 # extractable English language /watch/ URLs
1430                 if re.match(self._VPAGE_URL, url) is None:
1431                         request = urllib2.Request(url)
1432                         try:
1433                                 webpage = urllib2.urlopen(request).read()
1434                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1435                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1436                                 return
1437
1438                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1439                         if mobj is None:
1440                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
1441                                 return
1442                         yahoo_id = mobj.group(1)
1443
1444                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1445                         if mobj is None:
1446                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1447                                 return
1448                         yahoo_vid = mobj.group(1)
1449
1450                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1451                         return self._real_extract(url, new_video=False)
1452
1453                 # Retrieve video webpage to extract further information
1454                 request = urllib2.Request(url)
1455                 try:
1456                         self.report_download_webpage(video_id)
1457                         webpage = urllib2.urlopen(request).read()
1458                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1459                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1460                         return
1461
1462                 # Extract uploader and title from webpage
1463                 self.report_extraction(video_id)
1464                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1465                 if mobj is None:
1466                         self._downloader.trouble(u'ERROR: unable to extract video title')
1467                         return
1468                 video_title = mobj.group(1).decode('utf-8')
1469                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1470
1471                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1472                 if mobj is None:
1473                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1474                         return
1475                 video_uploader = mobj.group(1).decode('utf-8')
1476
1477                 # Extract video thumbnail
1478                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1479                 if mobj is None:
1480                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1481                         return
1482                 video_thumbnail = mobj.group(1).decode('utf-8')
1483
1484                 # Extract video description
1485                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1486                 if mobj is None:
1487                         self._downloader.trouble(u'ERROR: unable to extract video description')
1488                         return
1489                 video_description = mobj.group(1).decode('utf-8')
1490                 if not video_description: video_description = 'No description available.'
1491
1492                 # Extract video height and width
1493                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1494                 if mobj is None:
1495                         self._downloader.trouble(u'ERROR: unable to extract video height')
1496                         return
1497                 yv_video_height = mobj.group(1)
1498
1499                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1500                 if mobj is None:
1501                         self._downloader.trouble(u'ERROR: unable to extract video width')
1502                         return
1503                 yv_video_width = mobj.group(1)
1504
1505                 # Retrieve video playlist to extract media URL
1506                 # I'm not completely sure what all these options are, but we
1507                 # seem to need most of them, otherwise the server sends a 401.
1508                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1509                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1510                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1511                                           '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1512                                           '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1513                 try:
1514                         self.report_download_webpage(video_id)
1515                         webpage = urllib2.urlopen(request).read()
1516                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1517                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1518                         return
1519
1520                 # Extract media URL from playlist XML
1521                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1522                 if mobj is None:
1523                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1524                         return
1525                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1526                 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1527
1528                 try:
1529                         # Process video information
1530                         self._downloader.process_info({
1531                                 'id':           video_id.decode('utf-8'),
1532                                 'url':          video_url,
1533                                 'uploader':     video_uploader,
1534                                 'title':        video_title,
1535                                 'stitle':       simple_title,
1536                                 'ext':          video_extension.decode('utf-8'),
1537                                 'thumbnail':    video_thumbnail.decode('utf-8'),
1538                                 'description':  video_description,
1539                                 'thumbnail':    video_thumbnail,
1540                                 'description':  video_description,
1541                                 'player_url':   None,
1542                         })
1543                 except UnavailableFormatError:
1544                         self._downloader.trouble(u'ERROR: format not available for video')
1545
1546
1547 class GenericIE(InfoExtractor):
1548         """Generic last-resort information extractor."""
1549
1550         def __init__(self, downloader=None):
1551                 InfoExtractor.__init__(self, downloader)
1552
1553         @staticmethod
1554         def suitable(url):
1555                 return True
1556
1557         def report_download_webpage(self, video_id):
1558                 """Report webpage download."""
1559                 self._downloader.to_stdout(u'WARNING: Falling back on generic information extractor.')
1560                 self._downloader.to_stdout(u'[generic] %s: Downloading webpage' % video_id)
1561
1562         def report_extraction(self, video_id):
1563                 """Report information extraction."""
1564                 self._downloader.to_stdout(u'[generic] %s: Extracting information' % video_id)
1565
1566         def _real_initialize(self):
1567                 return
1568
1569         def _real_extract(self, url):
1570                 # At this point we have a new video
1571                 if self._downloader is not None:
1572                         self._downloader.increment_downloads()
1573
1574                 video_id = url.split('/')[-1]
1575                 request = urllib2.Request(url)
1576                 try:
1577                         self.report_download_webpage(video_id)
1578                         webpage = urllib2.urlopen(request).read()
1579                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1580                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1581                         return
1582                 except ValueError, err:
1583                         # since this is the last-resort InfoExtractor, if
1584                         # this error is thrown, it'll be thrown here
1585                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1586                         return
1587
1588                 # Start with something easy: JW Player in SWFObject
1589                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1590                 if mobj is None:
1591                         # Broaden the search a little bit
1592                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1593                 if mobj is None:
1594                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1595                         return
1596
1597                 # It's possible that one of the regexes
1598                 # matched, but returned an empty group:
1599                 if mobj.group(1) is None:
1600                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1601                         return
1602
1603                 video_url = urllib.unquote(mobj.group(1))
1604                 video_id  = os.path.basename(video_url)
1605
1606                 # here's a fun little line of code for you:
1607                 video_extension = os.path.splitext(video_id)[1][1:]
1608                 video_id        = os.path.splitext(video_id)[0]
1609
1610                 # it's tempting to parse this further, but you would
1611                 # have to take into account all the variations like
1612                 #   Video Title - Site Name
1613                 #   Site Name | Video Title
1614                 #   Video Title - Tagline | Site Name
1615                 # and so on and so forth; it's just not practical
1616                 mobj = re.search(r'<title>(.*)</title>', webpage)
1617                 if mobj is None:
1618                         self._downloader.trouble(u'ERROR: unable to extract title')
1619                         return
1620                 video_title = mobj.group(1).decode('utf-8')
1621                 video_title = sanitize_title(video_title)
1622                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1623
1624                 # video uploader is domain name
1625                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1626                 if mobj is None:
1627                         self._downloader.trouble(u'ERROR: unable to extract title')
1628                         return
1629                 video_uploader = mobj.group(1).decode('utf-8')
1630
1631                 try:
1632                         # Process video information
1633                         self._downloader.process_info({
1634                                 'id':           video_id.decode('utf-8'),
1635                                 'url':          video_url.decode('utf-8'),
1636                                 'uploader':     video_uploader,
1637                                 'title':        video_title,
1638                                 'stitle':       simple_title,
1639                                 'ext':          video_extension.decode('utf-8'),
1640                                 'format':       u'NA',
1641                                 'player_url':   None,
1642                         })
1643                 except UnavailableFormatError:
1644                         self._downloader.trouble(u'ERROR: format not available for video')
1645
1646
1647 class YoutubeSearchIE(InfoExtractor):
1648         """Information Extractor for YouTube search queries."""
1649         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1650         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1651         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1652         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1653         _youtube_ie = None
1654         _max_youtube_results = 1000
1655
1656         def __init__(self, youtube_ie, downloader=None):
1657                 InfoExtractor.__init__(self, downloader)
1658                 self._youtube_ie = youtube_ie
1659
1660         @staticmethod
1661         def suitable(url):
1662                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1663
1664         def report_download_page(self, query, pagenum):
1665                 """Report attempt to download playlist page with given number."""
1666                 query = query.decode(preferredencoding())
1667                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1668
1669         def _real_initialize(self):
1670                 self._youtube_ie.initialize()
1671
1672         def _real_extract(self, query):
1673                 mobj = re.match(self._VALID_QUERY, query)
1674                 if mobj is None:
1675                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1676                         return
1677
1678                 prefix, query = query.split(':')
1679                 prefix = prefix[8:]
1680                 query  = query.encode('utf-8')
1681                 if prefix == '':
1682                         self._download_n_results(query, 1)
1683                         return
1684                 elif prefix == 'all':
1685                         self._download_n_results(query, self._max_youtube_results)
1686                         return
1687                 else:
1688                         try:
1689                                 n = long(prefix)
1690                                 if n <= 0:
1691                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1692                                         return
1693                                 elif n > self._max_youtube_results:
1694                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
1695                                         n = self._max_youtube_results
1696                                 self._download_n_results(query, n)
1697                                 return
1698                         except ValueError: # parsing prefix as integer fails
1699                                 self._download_n_results(query, 1)
1700                                 return
1701
1702         def _download_n_results(self, query, n):
1703                 """Downloads a specified number of results for a query"""
1704
1705                 video_ids = []
1706                 already_seen = set()
1707                 pagenum = 1
1708
1709                 while True:
1710                         self.report_download_page(query, pagenum)
1711                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1712                         request = urllib2.Request(result_url, None, std_headers)
1713                         try:
1714                                 page = urllib2.urlopen(request).read()
1715                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1716                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1717                                 return
1718
1719                         # Extract video identifiers
1720                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1721                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1722                                 if video_id not in already_seen:
1723                                         video_ids.append(video_id)
1724                                         already_seen.add(video_id)
1725                                         if len(video_ids) == n:
1726                                                 # Specified n videos reached
1727                                                 for id in video_ids:
1728                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1729                                                 return
1730
1731                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1732                                 for id in video_ids:
1733                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1734                                 return
1735
1736                         pagenum = pagenum + 1
1737
1738 class GoogleSearchIE(InfoExtractor):
1739         """Information Extractor for Google Video search queries."""
1740         _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
1741         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1742         _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
1743         _MORE_PAGES_INDICATOR = r'<span>Next</span>'
1744         _google_ie = None
1745         _max_google_results = 1000
1746
1747         def __init__(self, google_ie, downloader=None):
1748                 InfoExtractor.__init__(self, downloader)
1749                 self._google_ie = google_ie
1750
1751         @staticmethod
1752         def suitable(url):
1753                 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
1754
1755         def report_download_page(self, query, pagenum):
1756                 """Report attempt to download playlist page with given number."""
1757                 query = query.decode(preferredencoding())
1758                 self._downloader.to_stdout(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1759
1760         def _real_initialize(self):
1761                 self._google_ie.initialize()
1762
1763         def _real_extract(self, query):
1764                 mobj = re.match(self._VALID_QUERY, query)
1765                 if mobj is None:
1766                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1767                         return
1768
1769                 prefix, query = query.split(':')
1770                 prefix = prefix[8:]
1771                 query  = query.encode('utf-8')
1772                 if prefix == '':
1773                         self._download_n_results(query, 1)
1774                         return
1775                 elif prefix == 'all':
1776                         self._download_n_results(query, self._max_google_results)
1777                         return
1778                 else:
1779                         try:
1780                                 n = long(prefix)
1781                                 if n <= 0:
1782                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1783                                         return
1784                                 elif n > self._max_google_results:
1785                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)'  % (self._max_google_results, n))
1786                                         n = self._max_google_results
1787                                 self._download_n_results(query, n)
1788                                 return
1789                         except ValueError: # parsing prefix as integer fails
1790                                 self._download_n_results(query, 1)
1791                                 return
1792
1793         def _download_n_results(self, query, n):
1794                 """Downloads a specified number of results for a query"""
1795
1796                 video_ids = []
1797                 already_seen = set()
1798                 pagenum = 1
1799
1800                 while True:
1801                         self.report_download_page(query, pagenum)
1802                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1803                         request = urllib2.Request(result_url, None, std_headers)
1804                         try:
1805                                 page = urllib2.urlopen(request).read()
1806                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1807                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1808                                 return
1809
1810                         # Extract video identifiers
1811                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1812                                 video_id = mobj.group(1)
1813                                 if video_id not in already_seen:
1814                                         video_ids.append(video_id)
1815                                         already_seen.add(video_id)
1816                                         if len(video_ids) == n:
1817                                                 # Specified n videos reached
1818                                                 for id in video_ids:
1819                                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1820                                                 return
1821
1822                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1823                                 for id in video_ids:
1824                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1825                                 return
1826
1827                         pagenum = pagenum + 1
1828
1829 class YahooSearchIE(InfoExtractor):
1830         """Information Extractor for Yahoo! Video search queries."""
1831         _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
1832         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1833         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1834         _MORE_PAGES_INDICATOR = r'\s*Next'
1835         _yahoo_ie = None
1836         _max_yahoo_results = 1000
1837
1838         def __init__(self, yahoo_ie, downloader=None):
1839                 InfoExtractor.__init__(self, downloader)
1840                 self._yahoo_ie = yahoo_ie
1841
1842         @staticmethod
1843         def suitable(url):
1844                 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
1845
1846         def report_download_page(self, query, pagenum):
1847                 """Report attempt to download playlist page with given number."""
1848                 query = query.decode(preferredencoding())
1849                 self._downloader.to_stdout(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1850
1851         def _real_initialize(self):
1852                 self._yahoo_ie.initialize()
1853
1854         def _real_extract(self, query):
1855                 mobj = re.match(self._VALID_QUERY, query)
1856                 if mobj is None:
1857                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1858                         return
1859
1860                 prefix, query = query.split(':')
1861                 prefix = prefix[8:]
1862                 query  = query.encode('utf-8')
1863                 if prefix == '':
1864                         self._download_n_results(query, 1)
1865                         return
1866                 elif prefix == 'all':
1867                         self._download_n_results(query, self._max_yahoo_results)
1868                         return
1869                 else:
1870                         try:
1871                                 n = long(prefix)
1872                                 if n <= 0:
1873                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1874                                         return
1875                                 elif n > self._max_yahoo_results:
1876                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)'  % (self._max_yahoo_results, n))
1877                                         n = self._max_yahoo_results
1878                                 self._download_n_results(query, n)
1879                                 return
1880                         except ValueError: # parsing prefix as integer fails
1881                                 self._download_n_results(query, 1)
1882                                 return
1883
1884         def _download_n_results(self, query, n):
1885                 """Downloads a specified number of results for a query"""
1886
1887                 video_ids = []
1888                 already_seen = set()
1889                 pagenum = 1
1890
1891                 while True:
1892                         self.report_download_page(query, pagenum)
1893                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1894                         request = urllib2.Request(result_url, None, std_headers)
1895                         try:
1896                                 page = urllib2.urlopen(request).read()
1897                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1898                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1899                                 return
1900
1901                         # Extract video identifiers
1902                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1903                                 video_id = mobj.group(1)
1904                                 if video_id not in already_seen:
1905                                         video_ids.append(video_id)
1906                                         already_seen.add(video_id)
1907                                         if len(video_ids) == n:
1908                                                 # Specified n videos reached
1909                                                 for id in video_ids:
1910                                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1911                                                 return
1912
1913                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1914                                 for id in video_ids:
1915                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1916                                 return
1917
1918                         pagenum = pagenum + 1
1919
1920 class YoutubePlaylistIE(InfoExtractor):
1921         """Information Extractor for YouTube playlists."""
1922
1923         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
1924         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1925         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1926         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1927         _youtube_ie = None
1928
1929         def __init__(self, youtube_ie, downloader=None):
1930                 InfoExtractor.__init__(self, downloader)
1931                 self._youtube_ie = youtube_ie
1932
1933         @staticmethod
1934         def suitable(url):
1935                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1936
1937         def report_download_page(self, playlist_id, pagenum):
1938                 """Report attempt to download playlist page with given number."""
1939                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1940
1941         def _real_initialize(self):
1942                 self._youtube_ie.initialize()
1943
1944         def _real_extract(self, url):
1945                 # Extract playlist id
1946                 mobj = re.match(self._VALID_URL, url)
1947                 if mobj is None:
1948                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1949                         return
1950
1951                 # Download playlist pages
1952                 playlist_id = mobj.group(1)
1953                 video_ids = []
1954                 pagenum = 1
1955
1956                 while True:
1957                         self.report_download_page(playlist_id, pagenum)
1958                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1959                         try:
1960                                 page = urllib2.urlopen(request).read()
1961                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1962                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1963                                 return
1964
1965                         # Extract video identifiers
1966                         ids_in_page = []
1967                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1968                                 if mobj.group(1) not in ids_in_page:
1969                                         ids_in_page.append(mobj.group(1))
1970                         video_ids.extend(ids_in_page)
1971
1972                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1973                                 break
1974                         pagenum = pagenum + 1
1975
1976                 for id in video_ids:
1977                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1978                 return
1979
1980 class YoutubeUserIE(InfoExtractor):
1981         """Information Extractor for YouTube users."""
1982
1983         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
1984         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1985         _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
1986         _youtube_ie = None
1987
1988         def __init__(self, youtube_ie, downloader=None):
1989                 InfoExtractor.__init__(self, downloader)
1990                 self._youtube_ie = youtube_ie
1991
1992         @staticmethod
1993         def suitable(url):
1994                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
1995
1996         def report_download_page(self, username):
1997                 """Report attempt to download user page."""
1998                 self._downloader.to_stdout(u'[youtube] user %s: Downloading page ' % (username))
1999
2000         def _real_initialize(self):
2001                 self._youtube_ie.initialize()
2002
2003         def _real_extract(self, url):
2004                 # Extract username
2005                 mobj = re.match(self._VALID_URL, url)
2006                 if mobj is None:
2007                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2008                         return
2009
2010                 # Download user page
2011                 username = mobj.group(1)
2012                 video_ids = []
2013                 pagenum = 1
2014
2015                 self.report_download_page(username)
2016                 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
2017                 try:
2018                         page = urllib2.urlopen(request).read()
2019                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2020                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2021                         return
2022
2023                 # Extract video identifiers
2024                 ids_in_page = []
2025
2026                 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2027                         if mobj.group(1) not in ids_in_page:
2028                                 ids_in_page.append(mobj.group(1))
2029                 video_ids.extend(ids_in_page)
2030
2031                 for id in video_ids:
2032                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2033                 return
2034
2035 class PostProcessor(object):
2036         """Post Processor class.
2037
2038         PostProcessor objects can be added to downloaders with their
2039         add_post_processor() method. When the downloader has finished a
2040         successful download, it will take its internal chain of PostProcessors
2041         and start calling the run() method on each one of them, first with
2042         an initial argument and then with the returned value of the previous
2043         PostProcessor.
2044
2045         The chain will be stopped if one of them ever returns None or the end
2046         of the chain is reached.
2047
2048         PostProcessor objects follow a "mutual registration" process similar
2049         to InfoExtractor objects.
2050         """
2051
2052         _downloader = None
2053
2054         def __init__(self, downloader=None):
2055                 self._downloader = downloader
2056
2057         def set_downloader(self, downloader):
2058                 """Sets the downloader for this PP."""
2059                 self._downloader = downloader
2060
2061         def run(self, information):
2062                 """Run the PostProcessor.
2063
2064                 The "information" argument is a dictionary like the ones
2065                 composed by InfoExtractors. The only difference is that this
2066                 one has an extra field called "filepath" that points to the
2067                 downloaded file.
2068
2069                 When this method returns None, the postprocessing chain is
2070                 stopped. However, this method may return an information
2071                 dictionary that will be passed to the next postprocessing
2072                 object in the chain. It can be the one it received after
2073                 changing some fields.
2074
2075                 In addition, this method may raise a PostProcessingError
2076                 exception that will be taken into account by the downloader
2077                 it was called from.
2078                 """
2079                 return information # by default, do nothing
2080
2081 ### MAIN PROGRAM ###
2082 if __name__ == '__main__':
2083         try:
2084                 # Modules needed only when running the main program
2085                 import getpass
2086                 import optparse
2087
2088                 # Function to update the program file with the latest version from bitbucket.org
2089                 def update_self(downloader, filename):
2090                         # Note: downloader only used for options
2091                         if not os.access (filename, os.W_OK):
2092                                 sys.exit('ERROR: no write permissions on %s' % filename)
2093
2094                         downloader.to_stdout('Updating to latest stable version...')
2095                         latest_url = 'http://bitbucket.org/rg3/youtube-dl/raw/tip/LATEST_VERSION'
2096                         latest_version = urllib.urlopen(latest_url).read().strip()
2097                         prog_url = 'http://bitbucket.org/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
2098                         newcontent = urllib.urlopen(prog_url).read()
2099                         stream = open(filename, 'w')
2100                         stream.write(newcontent)
2101                         stream.close()
2102                         downloader.to_stdout('Updated to version %s' % latest_version)
2103
2104                 # General configuration
2105                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
2106                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
2107                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
2108
2109                 # Parse command line
2110                 parser = optparse.OptionParser(
2111                         usage='Usage: %prog [options] url...',
2112                         version='2010.07.14',
2113                         conflict_handler='resolve',
2114                 )
2115
2116                 parser.add_option('-h', '--help',
2117                                 action='help', help='print this help text and exit')
2118                 parser.add_option('-v', '--version',
2119                                 action='version', help='print program version and exit')
2120                 parser.add_option('-U', '--update',
2121                                 action='store_true', dest='update_self', help='update this program to latest stable version')
2122                 parser.add_option('-i', '--ignore-errors',
2123                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
2124                 parser.add_option('-r', '--rate-limit',
2125                                 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
2126                 parser.add_option('-R', '--retries',
2127                                 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
2128
2129                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
2130                 authentication.add_option('-u', '--username',
2131                                 dest='username', metavar='USERNAME', help='account username')
2132                 authentication.add_option('-p', '--password',
2133                                 dest='password', metavar='PASSWORD', help='account password')
2134                 authentication.add_option('-n', '--netrc',
2135                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
2136                 parser.add_option_group(authentication)
2137
2138                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
2139                 video_format.add_option('-f', '--format',
2140                                 action='store', dest='format', metavar='FORMAT', help='video format code')
2141                 video_format.add_option('-b', '--best-quality',
2142                                 action='store_const', dest='format', help='download the best quality video possible', const='0')
2143                 video_format.add_option('-m', '--mobile-version',
2144                                 action='store_const', dest='format', help='alias for -f 17', const='17')
2145                 video_format.add_option('-d', '--high-def',
2146                                 action='store_const', dest='format', help='alias for -f 22', const='22')
2147                 video_format.add_option('--all-formats',
2148                                 action='store_const', dest='format', help='download all available video formats', const='-1')
2149                 video_format.add_option('--max-quality',
2150                                 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format limit for -b')
2151                 parser.add_option_group(video_format)
2152
2153                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
2154                 verbosity.add_option('-q', '--quiet',
2155                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
2156                 verbosity.add_option('-s', '--simulate',
2157                                 action='store_true', dest='simulate', help='do not download video', default=False)
2158                 verbosity.add_option('-g', '--get-url',
2159                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
2160                 verbosity.add_option('-e', '--get-title',
2161                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
2162                 verbosity.add_option('--get-thumbnail',
2163                                 action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False)
2164                 verbosity.add_option('--get-description',
2165                                 action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
2166                 verbosity.add_option('--no-progress',
2167                                 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
2168                 parser.add_option_group(verbosity)
2169
2170                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
2171                 filesystem.add_option('-t', '--title',
2172                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
2173                 filesystem.add_option('-l', '--literal',
2174                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
2175                 filesystem.add_option('-o', '--output',
2176                                 dest='outtmpl', metavar='TEMPLATE', help='output filename template')
2177                 filesystem.add_option('-a', '--batch-file',
2178                                 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
2179                 filesystem.add_option('-w', '--no-overwrites',
2180                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
2181                 filesystem.add_option('-c', '--continue',
2182                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
2183                 parser.add_option_group(filesystem)
2184
2185                 (opts, args) = parser.parse_args()
2186
2187                 # Batch file verification
2188                 batchurls = []
2189                 if opts.batchfile is not None:
2190                         try:
2191                                 if opts.batchfile == '-':
2192                                         batchfd = sys.stdin
2193                                 else:
2194                                         batchfd = open(opts.batchfile, 'r')
2195                                 batchurls = batchfd.readlines()
2196                                 batchurls = [x.strip() for x in batchurls]
2197                                 batchurls = [x for x in batchurls if len(x) > 0]
2198                         except IOError:
2199                                 sys.exit(u'ERROR: batch file could not be read')
2200                 all_urls = batchurls + args
2201
2202                 # Conflicting, missing and erroneous options
2203                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
2204                         parser.error(u'using .netrc conflicts with giving username/password')
2205                 if opts.password is not None and opts.username is None:
2206                         parser.error(u'account username missing')
2207                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
2208                         parser.error(u'using output template conflicts with using title or literal title')
2209                 if opts.usetitle and opts.useliteral:
2210                         parser.error(u'using title conflicts with using literal title')
2211                 if opts.username is not None and opts.password is None:
2212                         opts.password = getpass.getpass(u'Type account password and press return:')
2213                 if opts.ratelimit is not None:
2214                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
2215                         if numeric_limit is None:
2216                                 parser.error(u'invalid rate limit specified')
2217                         opts.ratelimit = numeric_limit
2218                 if opts.retries is not None:
2219                         try:
2220                                 opts.retries = long(opts.retries)
2221                         except (TypeError, ValueError), err:
2222                                 parser.error(u'invalid retry count specified')
2223
2224                 # Information extractors
2225                 youtube_ie = YoutubeIE()
2226                 metacafe_ie = MetacafeIE(youtube_ie)
2227                 dailymotion_ie = DailymotionIE()
2228                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
2229                 youtube_user_ie = YoutubeUserIE(youtube_ie)
2230                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
2231                 google_ie = GoogleIE()
2232                 google_search_ie = GoogleSearchIE(google_ie)
2233                 photobucket_ie = PhotobucketIE()
2234                 yahoo_ie = YahooIE()
2235                 yahoo_search_ie = YahooSearchIE(yahoo_ie)
2236                 generic_ie = GenericIE()
2237
2238                 # File downloader
2239                 fd = FileDownloader({
2240                         'usenetrc': opts.usenetrc,
2241                         'username': opts.username,
2242                         'password': opts.password,
2243                         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2244                         'forceurl': opts.geturl,
2245                         'forcetitle': opts.gettitle,
2246                         'forcethumbnail': opts.getthumbnail,
2247                         'forcedescription': opts.getdescription,
2248                         'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2249                         'format': opts.format,
2250                         'format_limit': opts.format_limit,
2251                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
2252                                 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
2253                                 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
2254                                 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
2255                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
2256                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
2257                                 or u'%(id)s.%(ext)s'),
2258                         'ignoreerrors': opts.ignoreerrors,
2259                         'ratelimit': opts.ratelimit,
2260                         'nooverwrites': opts.nooverwrites,
2261                         'retries': opts.retries,
2262                         'continuedl': opts.continue_dl,
2263                         'noprogress': opts.noprogress,
2264                         })
2265                 fd.add_info_extractor(youtube_search_ie)
2266                 fd.add_info_extractor(youtube_pl_ie)
2267                 fd.add_info_extractor(youtube_user_ie)
2268                 fd.add_info_extractor(metacafe_ie)
2269                 fd.add_info_extractor(dailymotion_ie)
2270                 fd.add_info_extractor(youtube_ie)
2271                 fd.add_info_extractor(google_ie)
2272                 fd.add_info_extractor(google_search_ie)
2273                 fd.add_info_extractor(photobucket_ie)
2274                 fd.add_info_extractor(yahoo_ie)
2275                 fd.add_info_extractor(yahoo_search_ie)
2276
2277                 # This must come last since it's the
2278                 # fallback if none of the others work
2279                 fd.add_info_extractor(generic_ie)
2280
2281                 # Update version
2282                 if opts.update_self:
2283                         update_self(fd, sys.argv[0])
2284
2285                 # Maybe do nothing
2286                 if len(all_urls) < 1:
2287                         if not opts.update_self:
2288                                 parser.error(u'you must provide at least one URL')
2289                         else:
2290                                 sys.exit()
2291                 retcode = fd.download(all_urls)
2292                 sys.exit(retcode)
2293
2294         except DownloadError:
2295                 sys.exit(1)
2296         except SameFileError:
2297                 sys.exit(u'ERROR: fixed output name but more than one file to download')
2298         except KeyboardInterrupt:
2299                 sys.exit(u'\nERROR: Interrupted by user')