_ Git - youtube-dl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # Author: Benjamin Johnson
   6 # License: Public domain code
   7 import htmlentitydefs
   8 import httplib
   9 import locale
  10 import math
  11 import netrc
  12 import os
  13 import os.path
  14 import re
  15 import socket
  16 import string
  17 import subprocess
  18 import sys
  19 import time
  20 import urllib
  21 import urllib2
  22
  23 # parse_qs was moved from the cgi module to the urlparse module recently.
  24 try:
  25         from urlparse import parse_qs
  26 except ImportError:
  27         from cgi import parse_qs
  28
  29 std_headers = {
  30         'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.6) Gecko/20100627 Firefox/3.6.6',
  31         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  32         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  33         'Accept-Language': 'en-us,en;q=0.5',
  34 }
  35
  36 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  37
  38 def preferredencoding():
  39         """Get preferred encoding.
  40
  41         Returns the best encoding scheme for the system, based on
  42         locale.getpreferredencoding() and some further tweaks.
  43         """
  44         def yield_preferredencoding():
  45                 try:
  46                         pref = locale.getpreferredencoding()
  47                         u'TEST'.encode(pref)
  48                 except:
  49                         pref = 'UTF-8'
  50                 while True:
  51                         yield pref
  52         return yield_preferredencoding().next()
  53
  54 def htmlentity_transform(matchobj):
  55         """Transforms an HTML entity to a Unicode character.
  56
  57         This function receives a match object and is intended to be used with
  58         the re.sub() function.
  59         """
  60         entity = matchobj.group(1)
  61
  62         # Known non-numeric HTML entity
  63         if entity in htmlentitydefs.name2codepoint:
  64                 return unichr(htmlentitydefs.name2codepoint[entity])
  65
  66         # Unicode character
  67         mobj = re.match(ur'(?u)#(x?\d+)', entity)
  68         if mobj is not None:
  69                 numstr = mobj.group(1)
  70                 if numstr.startswith(u'x'):
  71                         base = 16
  72                         numstr = u'0%s' % numstr
  73                 else:
  74                         base = 10
  75                 return unichr(long(numstr, base))
  76
  77         # Unknown entity in name, return its literal representation
  78         return (u'&%s;' % entity)
  79
  80 def sanitize_title(utitle):
  81         """Sanitizes a video title so it could be used as part of a filename."""
  82         utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
  83         return utitle.replace(unicode(os.sep), u'%')
  84
  85 def sanitize_open(filename, open_mode):
  86         """Try to open the given filename, and slightly tweak it if this fails.
  87
  88         Attempts to open the given filename. If this fails, it tries to change
  89         the filename slightly, step by step, until it's either able to open it
  90         or it fails and raises a final exception, like the standard open()
  91         function.
  92
  93         It returns the tuple (stream, definitive_file_name).
  94         """
  95         try:
  96                 if filename == u'-':
  97                         return (sys.stdout, filename)
  98                 stream = open(filename, open_mode)
  99                 return (stream, filename)
 100         except (IOError, OSError), err:
 101                 # In case of error, try to remove win32 forbidden chars
 102                 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 103
 104                 # An exception here should be caught in the caller
 105                 stream = open(filename, open_mode)
 106                 return (stream, filename)
 107
 108
 109 class DownloadError(Exception):
 110         """Download Error exception.
 111
 112         This exception may be thrown by FileDownloader objects if they are not
 113         configured to continue on errors. They will contain the appropriate
 114         error message.
 115         """
 116         pass
 117
 118 class SameFileError(Exception):
 119         """Same File exception.
 120
 121         This exception will be thrown by FileDownloader objects if they detect
 122         multiple files would have to be downloaded to the same file on disk.
 123         """
 124         pass
 125
 126 class PostProcessingError(Exception):
 127         """Post Processing exception.
 128
 129         This exception may be raised by PostProcessor's .run() method to
 130         indicate an error in the postprocessing task.
 131         """
 132         pass
 133
 134 class UnavailableVideoError(Exception):
 135         """Unavailable Format exception.
 136
 137         This exception will be thrown when a video is requested
 138         in a format that is not available for that video.
 139         """
 140         pass
 141
 142 class ContentTooShortError(Exception):
 143         """Content Too Short exception.
 144
 145         This exception may be raised by FileDownloader objects when a file they
 146         download is too small for what the server announced first, indicating
 147         the connection was probably interrupted.
 148         """
 149         # Both in bytes
 150         downloaded = None
 151         expected = None
 152
 153         def __init__(self, downloaded, expected):
 154                 self.downloaded = downloaded
 155                 self.expected = expected
 156
 157 class FileDownloader(object):
 158         """File Downloader class.
 159
 160         File downloader objects are the ones responsible of downloading the
 161         actual video file and writing it to disk if the user has requested
 162         it, among some other tasks. In most cases there should be one per
 163         program. As, given a video URL, the downloader doesn't know how to
 164         extract all the needed information, task that InfoExtractors do, it
 165         has to pass the URL to one of them.
 166
 167         For this, file downloader objects have a method that allows
 168         InfoExtractors to be registered in a given order. When it is passed
 169         a URL, the file downloader handles it to the first InfoExtractor it
 170         finds that reports being able to handle it. The InfoExtractor extracts
 171         all the information about the video or videos the URL refers to, and
 172         asks the FileDownloader to process the video information, possibly
 173         downloading the video.
 174
 175         File downloaders accept a lot of parameters. In order not to saturate
 176         the object constructor with arguments, it receives a dictionary of
 177         options instead. These options are available through the params
 178         attribute for the InfoExtractors to use. The FileDownloader also
 179         registers itself as the downloader in charge for the InfoExtractors
 180         that are added to it, so this is a "mutual registration".
 181
 182         Available options:
 183
 184         username:       Username for authentication purposes.
 185         password:       Password for authentication purposes.
 186         usenetrc:       Use netrc for authentication instead.
 187         quiet:          Do not print messages to stdout.
 188         forceurl:       Force printing final URL.
 189         forcetitle:     Force printing title.
 190         simulate:       Do not download the video files.
 191         format:         Video format code.
 192         format_limit:   Highest quality format to try.
 193         outtmpl:        Template for output names.
 194         ignoreerrors:   Do not stop on download errors.
 195         ratelimit:      Download speed limit, in bytes/sec.
 196         nooverwrites:   Prevent overwriting files.
 197         retries:        Number of times to retry for HTTP error 503
 198         continuedl:     Try to continue downloads if possible.
 199         noprogress:     Do not print the progress bar.
 200         """
 201
 202         params = None
 203         _ies = []
 204         _pps = []
 205         _download_retcode = None
 206         _num_downloads = None
 207
 208         def __init__(self, params):
 209                 """Create a FileDownloader object with the given options."""
 210                 self._ies = []
 211                 self._pps = []
 212                 self._download_retcode = 0
 213                 self._num_downloads = 0
 214                 self.params = params
 215
 216         @staticmethod
 217         def pmkdir(filename):
 218                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 219                 components = filename.split(os.sep)
 220                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 221                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 222                 for dir in aggregate:
 223                         if not os.path.exists(dir):
 224                                 os.mkdir(dir)
 225
 226         @staticmethod
 227         def format_bytes(bytes):
 228                 if bytes is None:
 229                         return 'N/A'
 230                 if type(bytes) is str:
 231                         bytes = float(bytes)
 232                 if bytes == 0.0:
 233                         exponent = 0
 234                 else:
 235                         exponent = long(math.log(bytes, 1024.0))
 236                 suffix = 'bkMGTPEZY'[exponent]
 237                 converted = float(bytes) / float(1024**exponent)
 238                 return '%.2f%s' % (converted, suffix)
 239
 240         @staticmethod
 241         def calc_percent(byte_counter, data_len):
 242                 if data_len is None:
 243                         return '---.-%'
 244                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 245
 246         @staticmethod
 247         def calc_eta(start, now, total, current):
 248                 if total is None:
 249                         return '--:--'
 250                 dif = now - start
 251                 if current == 0 or dif < 0.001: # One millisecond
 252                         return '--:--'
 253                 rate = float(current) / dif
 254                 eta = long((float(total) - float(current)) / rate)
 255                 (eta_mins, eta_secs) = divmod(eta, 60)
 256                 if eta_mins > 99:
 257                         return '--:--'
 258                 return '%02d:%02d' % (eta_mins, eta_secs)
 259
 260         @staticmethod
 261         def calc_speed(start, now, bytes):
 262                 dif = now - start
 263                 if bytes == 0 or dif < 0.001: # One millisecond
 264                         return '%10s' % '---b/s'
 265                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 266
 267         @staticmethod
 268         def best_block_size(elapsed_time, bytes):
 269                 new_min = max(bytes / 2.0, 1.0)
 270                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 271                 if elapsed_time < 0.001:
 272                         return long(new_max)
 273                 rate = bytes / elapsed_time
 274                 if rate > new_max:
 275                         return long(new_max)
 276                 if rate < new_min:
 277                         return long(new_min)
 278                 return long(rate)
 279
 280         @staticmethod
 281         def parse_bytes(bytestr):
 282                 """Parse a string indicating a byte quantity into a long integer."""
 283                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 284                 if matchobj is None:
 285                         return None
 286                 number = float(matchobj.group(1))
 287                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 288                 return long(round(number * multiplier))
 289
 290         @staticmethod
 291         def verify_url(url):
 292                 """Verify a URL is valid and data could be downloaded. Return real data URL."""
 293                 request = urllib2.Request(url, None, std_headers)
 294                 data = urllib2.urlopen(request)
 295                 data.read(1)
 296                 url = data.geturl()
 297                 data.close()
 298                 return url
 299
 300         def add_info_extractor(self, ie):
 301                 """Add an InfoExtractor object to the end of the list."""
 302                 self._ies.append(ie)
 303                 ie.set_downloader(self)
 304
 305         def add_post_processor(self, pp):
 306                 """Add a PostProcessor object to the end of the chain."""
 307                 self._pps.append(pp)
 308                 pp.set_downloader(self)
 309
 310         def to_stdout(self, message, skip_eol=False, ignore_encoding_errors=False):
 311                 """Print message to stdout if not in quiet mode."""
 312                 try:
 313                         if not self.params.get('quiet', False):
 314                                 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
 315                         sys.stdout.flush()
 316                 except (UnicodeEncodeError), err:
 317                         if not ignore_encoding_errors:
 318                                 raise
 319
 320         def to_stderr(self, message):
 321                 """Print message to stderr."""
 322                 print >>sys.stderr, message.encode(preferredencoding())
 323
 324         def fixed_template(self):
 325                 """Checks if the output template is fixed."""
 326                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 327
 328         def trouble(self, message=None):
 329                 """Determine action to take when a download problem appears.
 330
 331                 Depending on if the downloader has been configured to ignore
 332                 download errors or not, this method may throw an exception or
 333                 not when errors are found, after printing the message.
 334                 """
 335                 if message is not None:
 336                         self.to_stderr(message)
 337                 if not self.params.get('ignoreerrors', False):
 338                         raise DownloadError(message)
 339                 self._download_retcode = 1
 340
 341         def slow_down(self, start_time, byte_counter):
 342                 """Sleep if the download speed is over the rate limit."""
 343                 rate_limit = self.params.get('ratelimit', None)
 344                 if rate_limit is None or byte_counter == 0:
 345                         return
 346                 now = time.time()
 347                 elapsed = now - start_time
 348                 if elapsed <= 0.0:
 349                         return
 350                 speed = float(byte_counter) / elapsed
 351                 if speed > rate_limit:
 352                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 353
 354         def report_destination(self, filename):
 355                 """Report destination filename."""
 356                 self.to_stdout(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
 357
 358         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 359                 """Report download progress."""
 360                 if self.params.get('noprogress', False):
 361                         return
 362                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 363                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 364
 365         def report_resuming_byte(self, resume_len):
 366                 """Report attemtp to resume at given byte."""
 367                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 368
 369         def report_retry(self, count, retries):
 370                 """Report retry in case of HTTP error 503"""
 371                 self.to_stdout(u'[download] Got HTTP error 503. Retrying (attempt %d of %d)...' % (count, retries))
 372
 373         def report_file_already_downloaded(self, file_name):
 374                 """Report file has already been fully downloaded."""
 375                 try:
 376                         self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 377                 except (UnicodeEncodeError), err:
 378                         self.to_stdout(u'[download] The file has already been downloaded')
 379
 380         def report_unable_to_resume(self):
 381                 """Report it was impossible to resume download."""
 382                 self.to_stdout(u'[download] Unable to resume')
 383
 384         def report_finish(self):
 385                 """Report download finished."""
 386                 if self.params.get('noprogress', False):
 387                         self.to_stdout(u'[download] Download completed')
 388                 else:
 389                         self.to_stdout(u'')
 390
 391         def increment_downloads(self):
 392                 """Increment the ordinal that assigns a number to each file."""
 393                 self._num_downloads += 1
 394
 395         def process_info(self, info_dict):
 396                 """Process a single dictionary returned by an InfoExtractor."""
 397                 # Do nothing else if in simulate mode
 398                 if self.params.get('simulate', False):
 399                         # Verify URL if it's an HTTP one
 400                         if info_dict['url'].startswith('http'):
 401                                 try:
 402                                         self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8')
 403                                 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 404                                         raise UnavailableVideoError
 405
 406                         # Forced printings
 407                         if self.params.get('forcetitle', False):
 408                                 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 409                         if self.params.get('forceurl', False):
 410                                 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 411                         if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 412                                 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
 413                         if self.params.get('forcedescription', False) and 'description' in info_dict:
 414                                 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 415
 416                         return
 417
 418                 try:
 419                         template_dict = dict(info_dict)
 420                         template_dict['epoch'] = unicode(long(time.time()))
 421                         template_dict['ord'] = unicode('%05d' % self._num_downloads)
 422                         filename = self.params['outtmpl'] % template_dict
 423                 except (ValueError, KeyError), err:
 424                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 425                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 426                         self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
 427                         return
 428
 429                 try:
 430                         self.pmkdir(filename)
 431                 except (OSError, IOError), err:
 432                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 433                         return
 434
 435                 try:
 436                         success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
 437                 except (OSError, IOError), err:
 438                         raise UnavailableVideoError
 439                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 440                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 441                         return
 442                 except (ContentTooShortError, ), err:
 443                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 444                         return
 445
 446                 if success:
 447                         try:
 448                                 self.post_process(filename, info_dict)
 449                         except (PostProcessingError), err:
 450                                 self.trouble('ERROR: postprocessing: %s' % str(err))
 451                                 return
 452
 453         def download(self, url_list):
 454                 """Download a given list of URLs."""
 455                 if len(url_list) > 1 and self.fixed_template():
 456                         raise SameFileError(self.params['outtmpl'])
 457
 458                 for url in url_list:
 459                         suitable_found = False
 460                         for ie in self._ies:
 461                                 # Go to next InfoExtractor if not suitable
 462                                 if not ie.suitable(url):
 463                                         continue
 464
 465                                 # Suitable InfoExtractor found
 466                                 suitable_found = True
 467
 468                                 # Extract information from URL and process it
 469                                 ie.extract(url)
 470
 471                                 # Suitable InfoExtractor had been found; go to next URL
 472                                 break
 473
 474                         if not suitable_found:
 475                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 476
 477                 return self._download_retcode
 478
 479         def post_process(self, filename, ie_info):
 480                 """Run the postprocessing chain on the given file."""
 481                 info = dict(ie_info)
 482                 info['filepath'] = filename
 483                 for pp in self._pps:
 484                         info = pp.run(info)
 485                         if info is None:
 486                                 break
 487
 488         def _download_with_rtmpdump(self, filename, url, player_url):
 489                 self.report_destination(filename)
 490
 491                 # Check for rtmpdump first
 492                 try:
 493                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 494                 except (OSError, IOError):
 495                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 496                         return False
 497
 498                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 499                 # the connection was interrumpted and resuming appears to be
 500                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 501                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename]
 502                 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
 503                 while retval == 2 or retval == 1:
 504                         prevsize = os.path.getsize(filename)
 505                         self.to_stdout(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 506                         time.sleep(5.0) # This seems to be needed
 507                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 508                         cursize = os.path.getsize(filename)
 509                         if prevsize == cursize and retval == 1:
 510                                 break
 511                 if retval == 0:
 512                         self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
 513                         return True
 514                 else:
 515                         self.trouble('\nERROR: rtmpdump exited with code %d' % retval)
 516                         return False
 517
 518         def _do_download(self, filename, url, player_url):
 519                 # Attempt to download using rtmpdump
 520                 if url.startswith('rtmp'):
 521                         return self._download_with_rtmpdump(filename, url, player_url)
 522
 523                 stream = None
 524                 open_mode = 'wb'
 525                 basic_request = urllib2.Request(url, None, std_headers)
 526                 request = urllib2.Request(url, None, std_headers)
 527
 528                 # Establish possible resume length
 529                 if os.path.isfile(filename):
 530                         resume_len = os.path.getsize(filename)
 531                 else:
 532                         resume_len = 0
 533
 534                 # Request parameters in case of being able to resume
 535                 if self.params.get('continuedl', False) and resume_len != 0:
 536                         self.report_resuming_byte(resume_len)
 537                         request.add_header('Range','bytes=%d-' % resume_len)
 538                         open_mode = 'ab'
 539
 540                 count = 0
 541                 retries = self.params.get('retries', 0)
 542                 while True:
 543                         # Establish connection
 544                         try:
 545                                 data = urllib2.urlopen(request)
 546                                 break
 547                         except (urllib2.HTTPError, ), err:
 548                                 if err.code == 503:
 549                                         # Retry in case of HTTP error 503
 550                                         count += 1
 551                                         if count <= retries:
 552                                                 self.report_retry(count, retries)
 553                                                 continue
 554                                 if err.code != 416: #  416 is 'Requested range not satisfiable'
 555                                         raise
 556                                 # Unable to resume
 557                                 data = urllib2.urlopen(basic_request)
 558                                 content_length = data.info()['Content-Length']
 559
 560                                 if content_length is not None and long(content_length) == resume_len:
 561                                         # Because the file had already been fully downloaded
 562                                         self.report_file_already_downloaded(filename)
 563                                         return True
 564                                 else:
 565                                         # Because the server didn't let us
 566                                         self.report_unable_to_resume()
 567                                         open_mode = 'wb'
 568
 569                 data_len = data.info().get('Content-length', None)
 570                 data_len_str = self.format_bytes(data_len)
 571                 byte_counter = 0
 572                 block_size = 1024
 573                 start = time.time()
 574                 while True:
 575                         # Download and write
 576                         before = time.time()
 577                         data_block = data.read(block_size)
 578                         after = time.time()
 579                         data_block_len = len(data_block)
 580                         if data_block_len == 0:
 581                                 break
 582                         byte_counter += data_block_len
 583
 584                         # Open file just in time
 585                         if stream is None:
 586                                 try:
 587                                         (stream, filename) = sanitize_open(filename, open_mode)
 588                                         self.report_destination(filename)
 589                                 except (OSError, IOError), err:
 590                                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 591                                         return False
 592                         try:
 593                                 stream.write(data_block)
 594                         except (IOError, OSError), err:
 595                                 self.trouble('\nERROR: unable to write data: %s' % str(err))
 596                         block_size = self.best_block_size(after - before, data_block_len)
 597
 598                         # Progress message
 599                         percent_str = self.calc_percent(byte_counter, data_len)
 600                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 601                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 602                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 603
 604                         # Apply rate limit
 605                         self.slow_down(start, byte_counter)
 606
 607                 self.report_finish()
 608                 if data_len is not None and str(byte_counter) != data_len:
 609                         raise ContentTooShortError(byte_counter, long(data_len))
 610                 return True
 611
 612 class InfoExtractor(object):
 613         """Information Extractor class.
 614
 615         Information extractors are the classes that, given a URL, extract
 616         information from the video (or videos) the URL refers to. This
 617         information includes the real video URL, the video title and simplified
 618         title, author and others. The information is stored in a dictionary
 619         which is then passed to the FileDownloader. The FileDownloader
 620         processes this information possibly downloading the video to the file
 621         system, among other possible outcomes. The dictionaries must include
 622         the following fields:
 623
 624         id:             Video identifier.
 625         url:            Final video URL.
 626         uploader:       Nickname of the video uploader.
 627         title:          Literal title.
 628         stitle:         Simplified title.
 629         ext:            Video filename extension.
 630         format:         Video format.
 631         player_url:     SWF Player URL (may be None).
 632
 633         The following fields are optional. Their primary purpose is to allow
 634         youtube-dl to serve as the backend for a video search function, such
 635         as the one in youtube2mp3.  They are only used when their respective
 636         forced printing functions are called:
 637
 638         thumbnail:      Full URL to a video thumbnail image.
 639         description:    One-line video description.
 640
 641         Subclasses of this one should re-define the _real_initialize() and
 642         _real_extract() methods, as well as the suitable() static method.
 643         Probably, they should also be instantiated and added to the main
 644         downloader.
 645         """
 646
 647         _ready = False
 648         _downloader = None
 649
 650         def __init__(self, downloader=None):
 651                 """Constructor. Receives an optional downloader."""
 652                 self._ready = False
 653                 self.set_downloader(downloader)
 654
 655         @staticmethod
 656         def suitable(url):
 657                 """Receives a URL and returns True if suitable for this IE."""
 658                 return False
 659
 660         def initialize(self):
 661                 """Initializes an instance (authentication, etc)."""
 662                 if not self._ready:
 663                         self._real_initialize()
 664                         self._ready = True
 665
 666         def extract(self, url):
 667                 """Extracts URL information and returns it in list of dicts."""
 668                 self.initialize()
 669                 return self._real_extract(url)
 670
 671         def set_downloader(self, downloader):
 672                 """Sets the downloader for this IE."""
 673                 self._downloader = downloader
 674
 675         def _real_initialize(self):
 676                 """Real initialization process. Redefine in subclasses."""
 677                 pass
 678
 679         def _real_extract(self, url):
 680                 """Real extraction process. Redefine in subclasses."""
 681                 pass
 682
 683 class YoutubeIE(InfoExtractor):
 684         """Information extractor for youtube.com."""
 685
 686         _VALID_URL = r'^((?:http://)?(?:youtu\.be/|(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?[\?#](?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
 687         _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 688         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 689         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 690         _NETRC_MACHINE = 'youtube'
 691         # Listed in order of priority for the -b option
 692         _available_formats = ['38', '37', '22', '45', '35', '34', '43', '18', '6', '5', '17', '13', None]
 693         _video_extensions = {
 694                 '13': '3gp',
 695                 '17': 'mp4',
 696                 '18': 'mp4',
 697                 '22': 'mp4',
 698                 '37': 'mp4',
 699                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 700                 '43': 'webm',
 701                 '45': 'webm',
 702         }
 703
 704         @staticmethod
 705         def suitable(url):
 706                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 707
 708         def report_lang(self):
 709                 """Report attempt to set language."""
 710                 self._downloader.to_stdout(u'[youtube] Setting language')
 711
 712         def report_login(self):
 713                 """Report attempt to log in."""
 714                 self._downloader.to_stdout(u'[youtube] Logging in')
 715
 716         def report_age_confirmation(self):
 717                 """Report attempt to confirm age."""
 718                 self._downloader.to_stdout(u'[youtube] Confirming age')
 719
 720         def report_video_webpage_download(self, video_id):
 721                 """Report attempt to download video webpage."""
 722                 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
 723
 724         def report_video_info_webpage_download(self, video_id):
 725                 """Report attempt to download video info webpage."""
 726                 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
 727
 728         def report_information_extraction(self, video_id):
 729                 """Report attempt to extract video information."""
 730                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 731
 732         def report_unavailable_format(self, video_id, format):
 733                 """Report extracted video URL."""
 734                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 735
 736         def report_rtmp_download(self):
 737                 """Indicate the download will use the RTMP protocol."""
 738                 self._downloader.to_stdout(u'[youtube] RTMP download detected')
 739
 740         def _real_initialize(self):
 741                 if self._downloader is None:
 742                         return
 743
 744                 username = None
 745                 password = None
 746                 downloader_params = self._downloader.params
 747
 748                 # Attempt to use provided username and password or .netrc data
 749                 if downloader_params.get('username', None) is not None:
 750                         username = downloader_params['username']
 751                         password = downloader_params['password']
 752                 elif downloader_params.get('usenetrc', False):
 753                         try:
 754                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 755                                 if info is not None:
 756                                         username = info[0]
 757                                         password = info[2]
 758                                 else:
 759                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 760                         except (IOError, netrc.NetrcParseError), err:
 761                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 762                                 return
 763
 764                 # Set language
 765                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 766                 try:
 767                         self.report_lang()
 768                         urllib2.urlopen(request).read()
 769                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 770                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 771                         return
 772
 773                 # No authentication to be performed
 774                 if username is None:
 775                         return
 776
 777                 # Log in
 778                 login_form = {
 779                                 'current_form': 'loginForm',
 780                                 'next':         '/',
 781                                 'action_login': 'Log In',
 782                                 'username':     username,
 783                                 'password':     password,
 784                                 }
 785                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 786                 try:
 787                         self.report_login()
 788                         login_results = urllib2.urlopen(request).read()
 789                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 790                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 791                                 return
 792                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 793                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 794                         return
 795
 796                 # Confirm age
 797                 age_form = {
 798                                 'next_url':             '/',
 799                                 'action_confirm':       'Confirm',
 800                                 }
 801                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 802                 try:
 803                         self.report_age_confirmation()
 804                         age_results = urllib2.urlopen(request).read()
 805                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 806                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 807                         return
 808
 809         def _real_extract(self, url):
 810                 # Extract video id from URL
 811                 mobj = re.match(self._VALID_URL, url)
 812                 if mobj is None:
 813                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 814                         return
 815
 816                 # At this point we have a new video
 817                 if self._downloader is not None:
 818                         self._downloader.increment_downloads()
 819                 video_id = mobj.group(2)
 820
 821                 # Downloader parameters
 822                 best_quality = False
 823                 all_formats = False
 824                 format_param = None
 825                 quality_index = 0
 826                 if self._downloader is not None:
 827                         params = self._downloader.params
 828                         format_param = params.get('format', None)
 829                         if format_param == '0':
 830                                 format_limit = params.get('format_limit', None)
 831                                 if format_limit is not None:
 832                                         try:
 833                                                 # Start at a different format if the user has limited the maximum quality
 834                                                 quality_index = self._available_formats.index(format_limit)
 835                                         except ValueError:
 836                                                 pass
 837                                 format_param = self._available_formats[quality_index]
 838                                 best_quality = True
 839                         elif format_param == '-1':
 840                                 format_param = self._available_formats[quality_index]
 841                                 all_formats = True
 842
 843                 while True:
 844                         # Extension
 845                         video_extension = self._video_extensions.get(format_param, 'flv')
 846
 847                         # Get video webpage
 848                         self.report_video_webpage_download(video_id)
 849                         request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id, None, std_headers)
 850                         try:
 851                                 video_webpage = urllib2.urlopen(request).read()
 852                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 853                                 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 854                                 return
 855
 856                         # Attempt to extract SWF player URL
 857                         mobj = re.search(r'swfConfig.*"(http://.*?watch.*?-.*?\.swf)"', video_webpage)
 858                         if mobj is not None:
 859                                 player_url = mobj.group(1)
 860                         else:
 861                                 player_url = None
 862
 863                         # Get video info
 864                         self.report_video_info_webpage_download(video_id)
 865                         for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
 866                                 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
 867                                                    % (video_id, el_type))
 868                                 request = urllib2.Request(video_info_url, None, std_headers)
 869                                 try:
 870                                         video_info_webpage = urllib2.urlopen(request).read()
 871                                         video_info = parse_qs(video_info_webpage)
 872                                         if 'token' in video_info:
 873                                                 break
 874                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 875                                         self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 876                                         return
 877                         self.report_information_extraction(video_id)
 878
 879                         # "t" param
 880                         if 'token' not in video_info:
 881                                 # Attempt to see if YouTube has issued an error message
 882                                 if 'reason' not in video_info:
 883                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
 884                                         stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
 885                                         stream.write(video_info_webpage)
 886                                         stream.close()
 887                                 else:
 888                                         reason = urllib.unquote_plus(video_info['reason'][0])
 889                                         self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
 890                                 return
 891                         token = urllib.unquote_plus(video_info['token'][0])
 892                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
 893                         if format_param is not None:
 894                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 895
 896                         # Check possible RTMP download
 897                         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 898                                 self.report_rtmp_download()
 899                                 video_real_url = video_info['conn'][0]
 900
 901                         # uploader
 902                         if 'author' not in video_info:
 903                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 904                                 return
 905                         video_uploader = urllib.unquote_plus(video_info['author'][0])
 906
 907                         # title
 908                         if 'title' not in video_info:
 909                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 910                                 return
 911                         video_title = urllib.unquote_plus(video_info['title'][0])
 912                         video_title = video_title.decode('utf-8')
 913                         video_title = sanitize_title(video_title)
 914
 915                         # simplified title
 916                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 917                         simple_title = simple_title.strip(ur'_')
 918
 919                         # thumbnail image
 920                         if 'thumbnail_url' not in video_info:
 921                                 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
 922                                 video_thumbnail = ''
 923                         else:   # don't panic if we can't find it
 924                                 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
 925
 926                         # description
 927                         video_description = 'No description available.'
 928                         if self._downloader.params.get('forcedescription', False):
 929                                 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
 930                                 if mobj is not None:
 931                                         video_description = mobj.group(1)
 932
 933                         try:
 934                                 # Process video information
 935                                 self._downloader.process_info({
 936                                         'id':           video_id.decode('utf-8'),
 937                                         'url':          video_real_url.decode('utf-8'),
 938                                         'uploader':     video_uploader.decode('utf-8'),
 939                                         'title':        video_title,
 940                                         'stitle':       simple_title,
 941                                         'ext':          video_extension.decode('utf-8'),
 942                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
 943                                         'thumbnail':    video_thumbnail.decode('utf-8'),
 944                                         'description':  video_description.decode('utf-8'),
 945                                         'player_url':   player_url,
 946                                 })
 947
 948                                 if all_formats:
 949                                         quality_index += 1
 950                                         if quality_index == len(self._available_formats):
 951                                                 # None left to get
 952                                                 return
 953                                         else:
 954                                                 format_param = self._available_formats[quality_index]
 955                                                 continue
 956                                 return
 957
 958                         except UnavailableFormatError, err:
 959                                 if best_quality or all_formats:
 960                                         quality_index += 1
 961                                         if quality_index == len(self._available_formats):
 962                                                 # I don't ever expect this to happen
 963                                                 if not all_formats:
 964                                                         self._downloader.trouble(u'ERROR: no known formats available for video')
 965                                                 return
 966                                         else:
 967                                                 self.report_unavailable_format(video_id, format_param)
 968                                                 format_param = self._available_formats[quality_index]
 969                                                 continue
 970                                 else:
 971                                         self._downloader.trouble('ERROR: format not available for video')
 972                                         return
 973
 974
 975 class MetacafeIE(InfoExtractor):
 976         """Information Extractor for metacafe.com."""
 977
 978         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 979         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 980         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 981         _youtube_ie = None
 982
 983         def __init__(self, youtube_ie, downloader=None):
 984                 InfoExtractor.__init__(self, downloader)
 985                 self._youtube_ie = youtube_ie
 986
 987         @staticmethod
 988         def suitable(url):
 989                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 990
 991         def report_disclaimer(self):
 992                 """Report disclaimer retrieval."""
 993                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 994
 995         def report_age_confirmation(self):
 996                 """Report attempt to confirm age."""
 997                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 998
 999         def report_download_webpage(self, video_id):
1000                 """Report webpage download."""
1001                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
1002
1003         def report_extraction(self, video_id):
1004                 """Report information extraction."""
1005                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
1006
1007         def _real_initialize(self):
1008                 # Retrieve disclaimer
1009                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
1010                 try:
1011                         self.report_disclaimer()
1012                         disclaimer = urllib2.urlopen(request).read()
1013                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1014                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1015                         return
1016
1017                 # Confirm age
1018                 disclaimer_form = {
1019                         'filters': '0',
1020                         'submit': "Continue - I'm over 18",
1021                         }
1022                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
1023                 try:
1024                         self.report_age_confirmation()
1025                         disclaimer = urllib2.urlopen(request).read()
1026                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1027                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1028                         return
1029
1030         def _real_extract(self, url):
1031                 # Extract id and simplified title from URL
1032                 mobj = re.match(self._VALID_URL, url)
1033                 if mobj is None:
1034                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1035                         return
1036
1037                 video_id = mobj.group(1)
1038
1039                 # Check if video comes from YouTube
1040                 mobj2 = re.match(r'^yt-(.*)$', video_id)
1041                 if mobj2 is not None:
1042                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1043                         return
1044
1045                 # At this point we have a new video
1046                 self._downloader.increment_downloads()
1047
1048                 simple_title = mobj.group(2).decode('utf-8')
1049                 video_extension = 'flv'
1050
1051                 # Retrieve video webpage to extract further information
1052                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1053                 try:
1054                         self.report_download_webpage(video_id)
1055                         webpage = urllib2.urlopen(request).read()
1056                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1057                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1058                         return
1059
1060                 # Extract URL, uploader and title from webpage
1061                 self.report_extraction(video_id)
1062                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1063                 if mobj is None:
1064                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1065                         return
1066                 mediaURL = urllib.unquote(mobj.group(1))
1067
1068                 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1069                 #if mobj is None:
1070                 #       self._downloader.trouble(u'ERROR: unable to extract gdaKey')
1071                 #       return
1072                 #gdaKey = mobj.group(1)
1073                 #
1074                 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1075
1076                 video_url = mediaURL
1077
1078                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1079                 if mobj is None:
1080                         self._downloader.trouble(u'ERROR: unable to extract title')
1081                         return
1082                 video_title = mobj.group(1).decode('utf-8')
1083                 video_title = sanitize_title(video_title)
1084
1085                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1086                 if mobj is None:
1087                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1088                         return
1089                 video_uploader = mobj.group(1)
1090
1091                 try:
1092                         # Process video information
1093                         self._downloader.process_info({
1094                                 'id':           video_id.decode('utf-8'),
1095                                 'url':          video_url.decode('utf-8'),
1096                                 'uploader':     video_uploader.decode('utf-8'),
1097                                 'title':        video_title,
1098                                 'stitle':       simple_title,
1099                                 'ext':          video_extension.decode('utf-8'),
1100                                 'format':       u'NA',
1101                                 'player_url':   None,
1102                         })
1103                 except UnavailableVideoError:
1104                         self._downloader.trouble(u'ERROR: unable to download video')
1105
1106
1107 class DailymotionIE(InfoExtractor):
1108         """Information Extractor for Dailymotion"""
1109
1110         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1111
1112         def __init__(self, downloader=None):
1113                 InfoExtractor.__init__(self, downloader)
1114
1115         @staticmethod
1116         def suitable(url):
1117                 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1118
1119         def report_download_webpage(self, video_id):
1120                 """Report webpage download."""
1121                 self._downloader.to_stdout(u'[dailymotion] %s: Downloading webpage' % video_id)
1122
1123         def report_extraction(self, video_id):
1124                 """Report information extraction."""
1125                 self._downloader.to_stdout(u'[dailymotion] %s: Extracting information' % video_id)
1126
1127         def _real_initialize(self):
1128                 return
1129
1130         def _real_extract(self, url):
1131                 # Extract id and simplified title from URL
1132                 mobj = re.match(self._VALID_URL, url)
1133                 if mobj is None:
1134                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1135                         return
1136
1137                 # At this point we have a new video
1138                 self._downloader.increment_downloads()
1139                 video_id = mobj.group(1)
1140
1141                 simple_title = mobj.group(2).decode('utf-8')
1142                 video_extension = 'flv'
1143
1144                 # Retrieve video webpage to extract further information
1145                 request = urllib2.Request(url)
1146                 try:
1147                         self.report_download_webpage(video_id)
1148                         webpage = urllib2.urlopen(request).read()
1149                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1150                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1151                         return
1152
1153                 # Extract URL, uploader and title from webpage
1154                 self.report_extraction(video_id)
1155                 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1156                 if mobj is None:
1157                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1158                         return
1159                 mediaURL = urllib.unquote(mobj.group(1))
1160
1161                 # if needed add http://www.dailymotion.com/ if relative URL
1162
1163                 video_url = mediaURL
1164
1165                 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1166                 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1167                 if mobj is None:
1168                         self._downloader.trouble(u'ERROR: unable to extract title')
1169                         return
1170                 video_title = mobj.group(1).decode('utf-8')
1171                 video_title = sanitize_title(video_title)
1172
1173                 mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a></div>', webpage)
1174                 if mobj is None:
1175                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1176                         return
1177                 video_uploader = mobj.group(1)
1178
1179                 try:
1180                         # Process video information
1181                         self._downloader.process_info({
1182                                 'id':           video_id.decode('utf-8'),
1183                                 'url':          video_url.decode('utf-8'),
1184                                 'uploader':     video_uploader.decode('utf-8'),
1185                                 'title':        video_title,
1186                                 'stitle':       simple_title,
1187                                 'ext':          video_extension.decode('utf-8'),
1188                                 'format':       u'NA',
1189                                 'player_url':   None,
1190                         })
1191                 except UnavailableVideoError:
1192                         self._downloader.trouble(u'ERROR: unable to download video')
1193
1194 class GoogleIE(InfoExtractor):
1195         """Information extractor for video.google.com."""
1196
1197         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1198
1199         def __init__(self, downloader=None):
1200                 InfoExtractor.__init__(self, downloader)
1201
1202         @staticmethod
1203         def suitable(url):
1204                 return (re.match(GoogleIE._VALID_URL, url) is not None)
1205
1206         def report_download_webpage(self, video_id):
1207                 """Report webpage download."""
1208                 self._downloader.to_stdout(u'[video.google] %s: Downloading webpage' % video_id)
1209
1210         def report_extraction(self, video_id):
1211                 """Report information extraction."""
1212                 self._downloader.to_stdout(u'[video.google] %s: Extracting information' % video_id)
1213
1214         def _real_initialize(self):
1215                 return
1216
1217         def _real_extract(self, url):
1218                 # Extract id from URL
1219                 mobj = re.match(self._VALID_URL, url)
1220                 if mobj is None:
1221                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1222                         return
1223
1224                 # At this point we have a new video
1225                 self._downloader.increment_downloads()
1226                 video_id = mobj.group(1)
1227
1228                 video_extension = 'mp4'
1229
1230                 # Retrieve video webpage to extract further information
1231                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1232                 try:
1233                         self.report_download_webpage(video_id)
1234                         webpage = urllib2.urlopen(request).read()
1235                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1236                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1237                         return
1238
1239                 # Extract URL, uploader, and title from webpage
1240                 self.report_extraction(video_id)
1241                 mobj = re.search(r"download_url:'([^']+)'", webpage)
1242                 if mobj is None:
1243                         video_extension = 'flv'
1244                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1245                 if mobj is None:
1246                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1247                         return
1248                 mediaURL = urllib.unquote(mobj.group(1))
1249                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1250                 mediaURL = mediaURL.replace('\\x26', '\x26')
1251
1252                 video_url = mediaURL
1253
1254                 mobj = re.search(r'<title>(.*)</title>', webpage)
1255                 if mobj is None:
1256                         self._downloader.trouble(u'ERROR: unable to extract title')
1257                         return
1258                 video_title = mobj.group(1).decode('utf-8')
1259                 video_title = sanitize_title(video_title)
1260                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1261
1262                 # Extract video description
1263                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1264                 if mobj is None:
1265                         self._downloader.trouble(u'ERROR: unable to extract video description')
1266                         return
1267                 video_description = mobj.group(1).decode('utf-8')
1268                 if not video_description:
1269                         video_description = 'No description available.'
1270
1271                 # Extract video thumbnail
1272                 if self._downloader.params.get('forcethumbnail', False):
1273                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1274                         try:
1275                                 webpage = urllib2.urlopen(request).read()
1276                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1277                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1278                                 return
1279                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1280                         if mobj is None:
1281                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1282                                 return
1283                         video_thumbnail = mobj.group(1)
1284                 else:   # we need something to pass to process_info
1285                         video_thumbnail = ''
1286
1287
1288                 try:
1289                         # Process video information
1290                         self._downloader.process_info({
1291                                 'id':           video_id.decode('utf-8'),
1292                                 'url':          video_url.decode('utf-8'),
1293                                 'uploader':     u'NA',
1294                                 'title':        video_title,
1295                                 'stitle':       simple_title,
1296                                 'ext':          video_extension.decode('utf-8'),
1297                                 'format':       u'NA',
1298                                 'player_url':   None,
1299                         })
1300                 except UnavailableVideoError:
1301                         self._downloader.trouble(u'ERROR: unable to download video')
1302
1303
1304 class PhotobucketIE(InfoExtractor):
1305         """Information extractor for photobucket.com."""
1306
1307         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1308
1309         def __init__(self, downloader=None):
1310                 InfoExtractor.__init__(self, downloader)
1311
1312         @staticmethod
1313         def suitable(url):
1314                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1315
1316         def report_download_webpage(self, video_id):
1317                 """Report webpage download."""
1318                 self._downloader.to_stdout(u'[photobucket] %s: Downloading webpage' % video_id)
1319
1320         def report_extraction(self, video_id):
1321                 """Report information extraction."""
1322                 self._downloader.to_stdout(u'[photobucket] %s: Extracting information' % video_id)
1323
1324         def _real_initialize(self):
1325                 return
1326
1327         def _real_extract(self, url):
1328                 # Extract id from URL
1329                 mobj = re.match(self._VALID_URL, url)
1330                 if mobj is None:
1331                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1332                         return
1333
1334                 # At this point we have a new video
1335                 self._downloader.increment_downloads()
1336                 video_id = mobj.group(1)
1337
1338                 video_extension = 'flv'
1339
1340                 # Retrieve video webpage to extract further information
1341                 request = urllib2.Request(url)
1342                 try:
1343                         self.report_download_webpage(video_id)
1344                         webpage = urllib2.urlopen(request).read()
1345                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1346                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1347                         return
1348
1349                 # Extract URL, uploader, and title from webpage
1350                 self.report_extraction(video_id)
1351                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1352                 if mobj is None:
1353                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1354                         return
1355                 mediaURL = urllib.unquote(mobj.group(1))
1356
1357                 video_url = mediaURL
1358
1359                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1360                 if mobj is None:
1361                         self._downloader.trouble(u'ERROR: unable to extract title')
1362                         return
1363                 video_title = mobj.group(1).decode('utf-8')
1364                 video_title = sanitize_title(video_title)
1365                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1366
1367                 video_uploader = mobj.group(2).decode('utf-8')
1368
1369                 try:
1370                         # Process video information
1371                         self._downloader.process_info({
1372                                 'id':           video_id.decode('utf-8'),
1373                                 'url':          video_url.decode('utf-8'),
1374                                 'uploader':     video_uploader,
1375                                 'title':        video_title,
1376                                 'stitle':       simple_title,
1377                                 'ext':          video_extension.decode('utf-8'),
1378                                 'format':       u'NA',
1379                                 'player_url':   None,
1380                         })
1381                 except UnavailableVideoError:
1382                         self._downloader.trouble(u'ERROR: unable to download video')
1383
1384
1385 class YahooIE(InfoExtractor):
1386         """Information extractor for video.yahoo.com."""
1387
1388         # _VALID_URL matches all Yahoo! Video URLs
1389         # _VPAGE_URL matches only the extractable '/watch/' URLs
1390         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1391         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1392
1393         def __init__(self, downloader=None):
1394                 InfoExtractor.__init__(self, downloader)
1395
1396         @staticmethod
1397         def suitable(url):
1398                 return (re.match(YahooIE._VALID_URL, url) is not None)
1399
1400         def report_download_webpage(self, video_id):
1401                 """Report webpage download."""
1402                 self._downloader.to_stdout(u'[video.yahoo] %s: Downloading webpage' % video_id)
1403
1404         def report_extraction(self, video_id):
1405                 """Report information extraction."""
1406                 self._downloader.to_stdout(u'[video.yahoo] %s: Extracting information' % video_id)
1407
1408         def _real_initialize(self):
1409                 return
1410
1411         def _real_extract(self, url, new_video=True):
1412                 # Extract ID from URL
1413                 mobj = re.match(self._VALID_URL, url)
1414                 if mobj is None:
1415                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1416                         return
1417
1418                 # At this point we have a new video
1419                 self._downloader.increment_downloads()
1420                 video_id = mobj.group(2)
1421                 video_extension = 'flv'
1422
1423                 # Rewrite valid but non-extractable URLs as
1424                 # extractable English language /watch/ URLs
1425                 if re.match(self._VPAGE_URL, url) is None:
1426                         request = urllib2.Request(url)
1427                         try:
1428                                 webpage = urllib2.urlopen(request).read()
1429                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1430                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1431                                 return
1432
1433                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1434                         if mobj is None:
1435                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
1436                                 return
1437                         yahoo_id = mobj.group(1)
1438
1439                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1440                         if mobj is None:
1441                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1442                                 return
1443                         yahoo_vid = mobj.group(1)
1444
1445                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1446                         return self._real_extract(url, new_video=False)
1447
1448                 # Retrieve video webpage to extract further information
1449                 request = urllib2.Request(url)
1450                 try:
1451                         self.report_download_webpage(video_id)
1452                         webpage = urllib2.urlopen(request).read()
1453                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1454                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1455                         return
1456
1457                 # Extract uploader and title from webpage
1458                 self.report_extraction(video_id)
1459                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1460                 if mobj is None:
1461                         self._downloader.trouble(u'ERROR: unable to extract video title')
1462                         return
1463                 video_title = mobj.group(1).decode('utf-8')
1464                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1465
1466                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1467                 if mobj is None:
1468                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
1469                         return
1470                 video_uploader = mobj.group(1).decode('utf-8')
1471
1472                 # Extract video thumbnail
1473                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1474                 if mobj is None:
1475                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1476                         return
1477                 video_thumbnail = mobj.group(1).decode('utf-8')
1478
1479                 # Extract video description
1480                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1481                 if mobj is None:
1482                         self._downloader.trouble(u'ERROR: unable to extract video description')
1483                         return
1484                 video_description = mobj.group(1).decode('utf-8')
1485                 if not video_description: video_description = 'No description available.'
1486
1487                 # Extract video height and width
1488                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1489                 if mobj is None:
1490                         self._downloader.trouble(u'ERROR: unable to extract video height')
1491                         return
1492                 yv_video_height = mobj.group(1)
1493
1494                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1495                 if mobj is None:
1496                         self._downloader.trouble(u'ERROR: unable to extract video width')
1497                         return
1498                 yv_video_width = mobj.group(1)
1499
1500                 # Retrieve video playlist to extract media URL
1501                 # I'm not completely sure what all these options are, but we
1502                 # seem to need most of them, otherwise the server sends a 401.
1503                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1504                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1505                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1506                                           '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1507                                           '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1508                 try:
1509                         self.report_download_webpage(video_id)
1510                         webpage = urllib2.urlopen(request).read()
1511                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1512                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1513                         return
1514
1515                 # Extract media URL from playlist XML
1516                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1517                 if mobj is None:
1518                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1519                         return
1520                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1521                 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1522
1523                 try:
1524                         # Process video information
1525                         self._downloader.process_info({
1526                                 'id':           video_id.decode('utf-8'),
1527                                 'url':          video_url,
1528                                 'uploader':     video_uploader,
1529                                 'title':        video_title,
1530                                 'stitle':       simple_title,
1531                                 'ext':          video_extension.decode('utf-8'),
1532                                 'thumbnail':    video_thumbnail.decode('utf-8'),
1533                                 'description':  video_description,
1534                                 'thumbnail':    video_thumbnail,
1535                                 'description':  video_description,
1536                                 'player_url':   None,
1537                         })
1538                 except UnavailableVideoError:
1539                         self._downloader.trouble(u'ERROR: unable to download video')
1540
1541
1542 class GenericIE(InfoExtractor):
1543         """Generic last-resort information extractor."""
1544
1545         def __init__(self, downloader=None):
1546                 InfoExtractor.__init__(self, downloader)
1547
1548         @staticmethod
1549         def suitable(url):
1550                 return True
1551
1552         def report_download_webpage(self, video_id):
1553                 """Report webpage download."""
1554                 self._downloader.to_stdout(u'WARNING: Falling back on generic information extractor.')
1555                 self._downloader.to_stdout(u'[generic] %s: Downloading webpage' % video_id)
1556
1557         def report_extraction(self, video_id):
1558                 """Report information extraction."""
1559                 self._downloader.to_stdout(u'[generic] %s: Extracting information' % video_id)
1560
1561         def _real_initialize(self):
1562                 return
1563
1564         def _real_extract(self, url):
1565                 # At this point we have a new video
1566                 self._downloader.increment_downloads()
1567
1568                 video_id = url.split('/')[-1]
1569                 request = urllib2.Request(url)
1570                 try:
1571                         self.report_download_webpage(video_id)
1572                         webpage = urllib2.urlopen(request).read()
1573                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1574                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1575                         return
1576                 except ValueError, err:
1577                         # since this is the last-resort InfoExtractor, if
1578                         # this error is thrown, it'll be thrown here
1579                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1580                         return
1581
1582                 # Start with something easy: JW Player in SWFObject
1583                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1584                 if mobj is None:
1585                         # Broaden the search a little bit
1586                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1587                 if mobj is None:
1588                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1589                         return
1590
1591                 # It's possible that one of the regexes
1592                 # matched, but returned an empty group:
1593                 if mobj.group(1) is None:
1594                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1595                         return
1596
1597                 video_url = urllib.unquote(mobj.group(1))
1598                 video_id  = os.path.basename(video_url)
1599
1600                 # here's a fun little line of code for you:
1601                 video_extension = os.path.splitext(video_id)[1][1:]
1602                 video_id        = os.path.splitext(video_id)[0]
1603
1604                 # it's tempting to parse this further, but you would
1605                 # have to take into account all the variations like
1606                 #   Video Title - Site Name
1607                 #   Site Name | Video Title
1608                 #   Video Title - Tagline | Site Name
1609                 # and so on and so forth; it's just not practical
1610                 mobj = re.search(r'<title>(.*)</title>', webpage)
1611                 if mobj is None:
1612                         self._downloader.trouble(u'ERROR: unable to extract title')
1613                         return
1614                 video_title = mobj.group(1).decode('utf-8')
1615                 video_title = sanitize_title(video_title)
1616                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1617
1618                 # video uploader is domain name
1619                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1620                 if mobj is None:
1621                         self._downloader.trouble(u'ERROR: unable to extract title')
1622                         return
1623                 video_uploader = mobj.group(1).decode('utf-8')
1624
1625                 try:
1626                         # Process video information
1627                         self._downloader.process_info({
1628                                 'id':           video_id.decode('utf-8'),
1629                                 'url':          video_url.decode('utf-8'),
1630                                 'uploader':     video_uploader,
1631                                 'title':        video_title,
1632                                 'stitle':       simple_title,
1633                                 'ext':          video_extension.decode('utf-8'),
1634                                 'format':       u'NA',
1635                                 'player_url':   None,
1636                         })
1637                 except UnavailableVideoError, err:
1638                         self._downloader.trouble(u'ERROR: unable to download video')
1639
1640
1641 class YoutubeSearchIE(InfoExtractor):
1642         """Information Extractor for YouTube search queries."""
1643         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1644         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1645         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1646         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1647         _youtube_ie = None
1648         _max_youtube_results = 1000
1649
1650         def __init__(self, youtube_ie, downloader=None):
1651                 InfoExtractor.__init__(self, downloader)
1652                 self._youtube_ie = youtube_ie
1653
1654         @staticmethod
1655         def suitable(url):
1656                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1657
1658         def report_download_page(self, query, pagenum):
1659                 """Report attempt to download playlist page with given number."""
1660                 query = query.decode(preferredencoding())
1661                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1662
1663         def _real_initialize(self):
1664                 self._youtube_ie.initialize()
1665
1666         def _real_extract(self, query):
1667                 mobj = re.match(self._VALID_QUERY, query)
1668                 if mobj is None:
1669                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1670                         return
1671
1672                 prefix, query = query.split(':')
1673                 prefix = prefix[8:]
1674                 query  = query.encode('utf-8')
1675                 if prefix == '':
1676                         self._download_n_results(query, 1)
1677                         return
1678                 elif prefix == 'all':
1679                         self._download_n_results(query, self._max_youtube_results)
1680                         return
1681                 else:
1682                         try:
1683                                 n = long(prefix)
1684                                 if n <= 0:
1685                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1686                                         return
1687                                 elif n > self._max_youtube_results:
1688                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
1689                                         n = self._max_youtube_results
1690                                 self._download_n_results(query, n)
1691                                 return
1692                         except ValueError: # parsing prefix as integer fails
1693                                 self._download_n_results(query, 1)
1694                                 return
1695
1696         def _download_n_results(self, query, n):
1697                 """Downloads a specified number of results for a query"""
1698
1699                 video_ids = []
1700                 already_seen = set()
1701                 pagenum = 1
1702
1703                 while True:
1704                         self.report_download_page(query, pagenum)
1705                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1706                         request = urllib2.Request(result_url, None, std_headers)
1707                         try:
1708                                 page = urllib2.urlopen(request).read()
1709                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1710                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1711                                 return
1712
1713                         # Extract video identifiers
1714                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1715                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1716                                 if video_id not in already_seen:
1717                                         video_ids.append(video_id)
1718                                         already_seen.add(video_id)
1719                                         if len(video_ids) == n:
1720                                                 # Specified n videos reached
1721                                                 for id in video_ids:
1722                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1723                                                 return
1724
1725                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1726                                 for id in video_ids:
1727                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1728                                 return
1729
1730                         pagenum = pagenum + 1
1731
1732 class GoogleSearchIE(InfoExtractor):
1733         """Information Extractor for Google Video search queries."""
1734         _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
1735         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1736         _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
1737         _MORE_PAGES_INDICATOR = r'<span>Next</span>'
1738         _google_ie = None
1739         _max_google_results = 1000
1740
1741         def __init__(self, google_ie, downloader=None):
1742                 InfoExtractor.__init__(self, downloader)
1743                 self._google_ie = google_ie
1744
1745         @staticmethod
1746         def suitable(url):
1747                 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
1748
1749         def report_download_page(self, query, pagenum):
1750                 """Report attempt to download playlist page with given number."""
1751                 query = query.decode(preferredencoding())
1752                 self._downloader.to_stdout(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1753
1754         def _real_initialize(self):
1755                 self._google_ie.initialize()
1756
1757         def _real_extract(self, query):
1758                 mobj = re.match(self._VALID_QUERY, query)
1759                 if mobj is None:
1760                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1761                         return
1762
1763                 prefix, query = query.split(':')
1764                 prefix = prefix[8:]
1765                 query  = query.encode('utf-8')
1766                 if prefix == '':
1767                         self._download_n_results(query, 1)
1768                         return
1769                 elif prefix == 'all':
1770                         self._download_n_results(query, self._max_google_results)
1771                         return
1772                 else:
1773                         try:
1774                                 n = long(prefix)
1775                                 if n <= 0:
1776                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1777                                         return
1778                                 elif n > self._max_google_results:
1779                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)'  % (self._max_google_results, n))
1780                                         n = self._max_google_results
1781                                 self._download_n_results(query, n)
1782                                 return
1783                         except ValueError: # parsing prefix as integer fails
1784                                 self._download_n_results(query, 1)
1785                                 return
1786
1787         def _download_n_results(self, query, n):
1788                 """Downloads a specified number of results for a query"""
1789
1790                 video_ids = []
1791                 already_seen = set()
1792                 pagenum = 1
1793
1794                 while True:
1795                         self.report_download_page(query, pagenum)
1796                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1797                         request = urllib2.Request(result_url, None, std_headers)
1798                         try:
1799                                 page = urllib2.urlopen(request).read()
1800                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1801                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1802                                 return
1803
1804                         # Extract video identifiers
1805                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1806                                 video_id = mobj.group(1)
1807                                 if video_id not in already_seen:
1808                                         video_ids.append(video_id)
1809                                         already_seen.add(video_id)
1810                                         if len(video_ids) == n:
1811                                                 # Specified n videos reached
1812                                                 for id in video_ids:
1813                                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1814                                                 return
1815
1816                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1817                                 for id in video_ids:
1818                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1819                                 return
1820
1821                         pagenum = pagenum + 1
1822
1823 class YahooSearchIE(InfoExtractor):
1824         """Information Extractor for Yahoo! Video search queries."""
1825         _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
1826         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1827         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1828         _MORE_PAGES_INDICATOR = r'\s*Next'
1829         _yahoo_ie = None
1830         _max_yahoo_results = 1000
1831
1832         def __init__(self, yahoo_ie, downloader=None):
1833                 InfoExtractor.__init__(self, downloader)
1834                 self._yahoo_ie = yahoo_ie
1835
1836         @staticmethod
1837         def suitable(url):
1838                 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
1839
1840         def report_download_page(self, query, pagenum):
1841                 """Report attempt to download playlist page with given number."""
1842                 query = query.decode(preferredencoding())
1843                 self._downloader.to_stdout(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1844
1845         def _real_initialize(self):
1846                 self._yahoo_ie.initialize()
1847
1848         def _real_extract(self, query):
1849                 mobj = re.match(self._VALID_QUERY, query)
1850                 if mobj is None:
1851                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1852                         return
1853
1854                 prefix, query = query.split(':')
1855                 prefix = prefix[8:]
1856                 query  = query.encode('utf-8')
1857                 if prefix == '':
1858                         self._download_n_results(query, 1)
1859                         return
1860                 elif prefix == 'all':
1861                         self._download_n_results(query, self._max_yahoo_results)
1862                         return
1863                 else:
1864                         try:
1865                                 n = long(prefix)
1866                                 if n <= 0:
1867                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1868                                         return
1869                                 elif n > self._max_yahoo_results:
1870                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)'  % (self._max_yahoo_results, n))
1871                                         n = self._max_yahoo_results
1872                                 self._download_n_results(query, n)
1873                                 return
1874                         except ValueError: # parsing prefix as integer fails
1875                                 self._download_n_results(query, 1)
1876                                 return
1877
1878         def _download_n_results(self, query, n):
1879                 """Downloads a specified number of results for a query"""
1880
1881                 video_ids = []
1882                 already_seen = set()
1883                 pagenum = 1
1884
1885                 while True:
1886                         self.report_download_page(query, pagenum)
1887                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1888                         request = urllib2.Request(result_url, None, std_headers)
1889                         try:
1890                                 page = urllib2.urlopen(request).read()
1891                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1892                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1893                                 return
1894
1895                         # Extract video identifiers
1896                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1897                                 video_id = mobj.group(1)
1898                                 if video_id not in already_seen:
1899                                         video_ids.append(video_id)
1900                                         already_seen.add(video_id)
1901                                         if len(video_ids) == n:
1902                                                 # Specified n videos reached
1903                                                 for id in video_ids:
1904                                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1905                                                 return
1906
1907                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1908                                 for id in video_ids:
1909                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1910                                 return
1911
1912                         pagenum = pagenum + 1
1913
1914 class YoutubePlaylistIE(InfoExtractor):
1915         """Information Extractor for YouTube playlists."""
1916
1917         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
1918         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1919         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1920         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1921         _youtube_ie = None
1922
1923         def __init__(self, youtube_ie, downloader=None):
1924                 InfoExtractor.__init__(self, downloader)
1925                 self._youtube_ie = youtube_ie
1926
1927         @staticmethod
1928         def suitable(url):
1929                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1930
1931         def report_download_page(self, playlist_id, pagenum):
1932                 """Report attempt to download playlist page with given number."""
1933                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1934
1935         def _real_initialize(self):
1936                 self._youtube_ie.initialize()
1937
1938         def _real_extract(self, url):
1939                 # Extract playlist id
1940                 mobj = re.match(self._VALID_URL, url)
1941                 if mobj is None:
1942                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1943                         return
1944
1945                 # Download playlist pages
1946                 playlist_id = mobj.group(1)
1947                 video_ids = []
1948                 pagenum = 1
1949
1950                 while True:
1951                         self.report_download_page(playlist_id, pagenum)
1952                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1953                         try:
1954                                 page = urllib2.urlopen(request).read()
1955                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1956                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1957                                 return
1958
1959                         # Extract video identifiers
1960                         ids_in_page = []
1961                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1962                                 if mobj.group(1) not in ids_in_page:
1963                                         ids_in_page.append(mobj.group(1))
1964                         video_ids.extend(ids_in_page)
1965
1966                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1967                                 break
1968                         pagenum = pagenum + 1
1969
1970                 for id in video_ids:
1971                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1972                 return
1973
1974 class YoutubeUserIE(InfoExtractor):
1975         """Information Extractor for YouTube users."""
1976
1977         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
1978         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1979         _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
1980         _youtube_ie = None
1981
1982         def __init__(self, youtube_ie, downloader=None):
1983                 InfoExtractor.__init__(self, downloader)
1984                 self._youtube_ie = youtube_ie
1985
1986         @staticmethod
1987         def suitable(url):
1988                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
1989
1990         def report_download_page(self, username):
1991                 """Report attempt to download user page."""
1992                 self._downloader.to_stdout(u'[youtube] user %s: Downloading page ' % (username))
1993
1994         def _real_initialize(self):
1995                 self._youtube_ie.initialize()
1996
1997         def _real_extract(self, url):
1998                 # Extract username
1999                 mobj = re.match(self._VALID_URL, url)
2000                 if mobj is None:
2001                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2002                         return
2003
2004                 # Download user page
2005                 username = mobj.group(1)
2006                 video_ids = []
2007                 pagenum = 1
2008
2009                 self.report_download_page(username)
2010                 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
2011                 try:
2012                         page = urllib2.urlopen(request).read()
2013                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2014                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2015                         return
2016
2017                 # Extract video identifiers
2018                 ids_in_page = []
2019
2020                 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2021                         if mobj.group(1) not in ids_in_page:
2022                                 ids_in_page.append(mobj.group(1))
2023                 video_ids.extend(ids_in_page)
2024
2025                 for id in video_ids:
2026                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2027                 return
2028
2029 class PostProcessor(object):
2030         """Post Processor class.
2031
2032         PostProcessor objects can be added to downloaders with their
2033         add_post_processor() method. When the downloader has finished a
2034         successful download, it will take its internal chain of PostProcessors
2035         and start calling the run() method on each one of them, first with
2036         an initial argument and then with the returned value of the previous
2037         PostProcessor.
2038
2039         The chain will be stopped if one of them ever returns None or the end
2040         of the chain is reached.
2041
2042         PostProcessor objects follow a "mutual registration" process similar
2043         to InfoExtractor objects.
2044         """
2045
2046         _downloader = None
2047
2048         def __init__(self, downloader=None):
2049                 self._downloader = downloader
2050
2051         def set_downloader(self, downloader):
2052                 """Sets the downloader for this PP."""
2053                 self._downloader = downloader
2054
2055         def run(self, information):
2056                 """Run the PostProcessor.
2057
2058                 The "information" argument is a dictionary like the ones
2059                 composed by InfoExtractors. The only difference is that this
2060                 one has an extra field called "filepath" that points to the
2061                 downloaded file.
2062
2063                 When this method returns None, the postprocessing chain is
2064                 stopped. However, this method may return an information
2065                 dictionary that will be passed to the next postprocessing
2066                 object in the chain. It can be the one it received after
2067                 changing some fields.
2068
2069                 In addition, this method may raise a PostProcessingError
2070                 exception that will be taken into account by the downloader
2071                 it was called from.
2072                 """
2073                 return information # by default, do nothing
2074
2075 ### MAIN PROGRAM ###
2076 if __name__ == '__main__':
2077         try:
2078                 # Modules needed only when running the main program
2079                 import getpass
2080                 import optparse
2081
2082                 # Function to update the program file with the latest version from bitbucket.org
2083                 def update_self(downloader, filename):
2084                         # Note: downloader only used for options
2085                         if not os.access (filename, os.W_OK):
2086                                 sys.exit('ERROR: no write permissions on %s' % filename)
2087
2088                         downloader.to_stdout('Updating to latest stable version...')
2089                         latest_url = 'http://bitbucket.org/rg3/youtube-dl/raw/tip/LATEST_VERSION'
2090                         latest_version = urllib.urlopen(latest_url).read().strip()
2091                         prog_url = 'http://bitbucket.org/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
2092                         newcontent = urllib.urlopen(prog_url).read()
2093                         stream = open(filename, 'w')
2094                         stream.write(newcontent)
2095                         stream.close()
2096                         downloader.to_stdout('Updated to version %s' % latest_version)
2097
2098                 # General configuration
2099                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
2100                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
2101                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
2102
2103                 # Parse command line
2104                 parser = optparse.OptionParser(
2105                         usage='Usage: %prog [options] url...',
2106                         version='2010.07.14',
2107                         conflict_handler='resolve',
2108                 )
2109
2110                 parser.add_option('-h', '--help',
2111                                 action='help', help='print this help text and exit')
2112                 parser.add_option('-v', '--version',
2113                                 action='version', help='print program version and exit')
2114                 parser.add_option('-U', '--update',
2115                                 action='store_true', dest='update_self', help='update this program to latest stable version')
2116                 parser.add_option('-i', '--ignore-errors',
2117                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
2118                 parser.add_option('-r', '--rate-limit',
2119                                 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
2120                 parser.add_option('-R', '--retries',
2121                                 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
2122
2123                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
2124                 authentication.add_option('-u', '--username',
2125                                 dest='username', metavar='USERNAME', help='account username')
2126                 authentication.add_option('-p', '--password',
2127                                 dest='password', metavar='PASSWORD', help='account password')
2128                 authentication.add_option('-n', '--netrc',
2129                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
2130                 parser.add_option_group(authentication)
2131
2132                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
2133                 video_format.add_option('-f', '--format',
2134                                 action='store', dest='format', metavar='FORMAT', help='video format code')
2135                 video_format.add_option('-m', '--mobile-version',
2136                                 action='store_const', dest='format', help='alias for -f 17', const='17')
2137                 video_format.add_option('--all-formats',
2138                                 action='store_const', dest='format', help='download all available video formats', const='-1')
2139                 video_format.add_option('--max-quality',
2140                                 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
2141                 parser.add_option_group(video_format)
2142
2143                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
2144                 verbosity.add_option('-q', '--quiet',
2145                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
2146                 verbosity.add_option('-s', '--simulate',
2147                                 action='store_true', dest='simulate', help='do not download video', default=False)
2148                 verbosity.add_option('-g', '--get-url',
2149                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
2150                 verbosity.add_option('-e', '--get-title',
2151                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
2152                 verbosity.add_option('--get-thumbnail',
2153                                 action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False)
2154                 verbosity.add_option('--get-description',
2155                                 action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
2156                 verbosity.add_option('--no-progress',
2157                                 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
2158                 parser.add_option_group(verbosity)
2159
2160                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
2161                 filesystem.add_option('-t', '--title',
2162                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
2163                 filesystem.add_option('-l', '--literal',
2164                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
2165                 filesystem.add_option('-o', '--output',
2166                                 dest='outtmpl', metavar='TEMPLATE', help='output filename template')
2167                 filesystem.add_option('-a', '--batch-file',
2168                                 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
2169                 filesystem.add_option('-w', '--no-overwrites',
2170                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
2171                 filesystem.add_option('-c', '--continue',
2172                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
2173                 parser.add_option_group(filesystem)
2174
2175                 (opts, args) = parser.parse_args()
2176
2177                 # Batch file verification
2178                 batchurls = []
2179                 if opts.batchfile is not None:
2180                         try:
2181                                 if opts.batchfile == '-':
2182                                         batchfd = sys.stdin
2183                                 else:
2184                                         batchfd = open(opts.batchfile, 'r')
2185                                 batchurls = batchfd.readlines()
2186                                 batchurls = [x.strip() for x in batchurls]
2187                                 batchurls = [x for x in batchurls if len(x) > 0]
2188                         except IOError:
2189                                 sys.exit(u'ERROR: batch file could not be read')
2190                 all_urls = batchurls + args
2191
2192                 # Conflicting, missing and erroneous options
2193                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
2194                         parser.error(u'using .netrc conflicts with giving username/password')
2195                 if opts.password is not None and opts.username is None:
2196                         parser.error(u'account username missing')
2197                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
2198                         parser.error(u'using output template conflicts with using title or literal title')
2199                 if opts.usetitle and opts.useliteral:
2200                         parser.error(u'using title conflicts with using literal title')
2201                 if opts.username is not None and opts.password is None:
2202                         opts.password = getpass.getpass(u'Type account password and press return:')
2203                 if opts.ratelimit is not None:
2204                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
2205                         if numeric_limit is None:
2206                                 parser.error(u'invalid rate limit specified')
2207                         opts.ratelimit = numeric_limit
2208                 if opts.retries is not None:
2209                         try:
2210                                 opts.retries = long(opts.retries)
2211                         except (TypeError, ValueError), err:
2212                                 parser.error(u'invalid retry count specified')
2213
2214                 # Information extractors
2215                 youtube_ie = YoutubeIE()
2216                 metacafe_ie = MetacafeIE(youtube_ie)
2217                 dailymotion_ie = DailymotionIE()
2218                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
2219                 youtube_user_ie = YoutubeUserIE(youtube_ie)
2220                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
2221                 google_ie = GoogleIE()
2222                 google_search_ie = GoogleSearchIE(google_ie)
2223                 photobucket_ie = PhotobucketIE()
2224                 yahoo_ie = YahooIE()
2225                 yahoo_search_ie = YahooSearchIE(yahoo_ie)
2226                 generic_ie = GenericIE()
2227
2228                 # File downloader
2229                 fd = FileDownloader({
2230                         'usenetrc': opts.usenetrc,
2231                         'username': opts.username,
2232                         'password': opts.password,
2233                         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2234                         'forceurl': opts.geturl,
2235                         'forcetitle': opts.gettitle,
2236                         'forcethumbnail': opts.getthumbnail,
2237                         'forcedescription': opts.getdescription,
2238                         'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2239                         'format': opts.format,
2240                         'format_limit': opts.format_limit,
2241                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
2242                                 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
2243                                 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
2244                                 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
2245                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
2246                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
2247                                 or u'%(id)s.%(ext)s'),
2248                         'ignoreerrors': opts.ignoreerrors,
2249                         'ratelimit': opts.ratelimit,
2250                         'nooverwrites': opts.nooverwrites,
2251                         'retries': opts.retries,
2252                         'continuedl': opts.continue_dl,
2253                         'noprogress': opts.noprogress,
2254                         })
2255                 fd.add_info_extractor(youtube_search_ie)
2256                 fd.add_info_extractor(youtube_pl_ie)
2257                 fd.add_info_extractor(youtube_user_ie)
2258                 fd.add_info_extractor(metacafe_ie)
2259                 fd.add_info_extractor(dailymotion_ie)
2260                 fd.add_info_extractor(youtube_ie)
2261                 fd.add_info_extractor(google_ie)
2262                 fd.add_info_extractor(google_search_ie)
2263                 fd.add_info_extractor(photobucket_ie)
2264                 fd.add_info_extractor(yahoo_ie)
2265                 fd.add_info_extractor(yahoo_search_ie)
2266
2267                 # This must come last since it's the
2268                 # fallback if none of the others work
2269                 fd.add_info_extractor(generic_ie)
2270
2271                 # Update version
2272                 if opts.update_self:
2273                         update_self(fd, sys.argv[0])
2274
2275                 # Maybe do nothing
2276                 if len(all_urls) < 1:
2277                         if not opts.update_self:
2278                                 parser.error(u'you must provide at least one URL')
2279                         else:
2280                                 sys.exit()
2281                 retcode = fd.download(all_urls)
2282                 sys.exit(retcode)
2283
2284         except DownloadError:
2285                 sys.exit(1)
2286         except SameFileError:
2287                 sys.exit(u'ERROR: fixed output name but more than one file to download')
2288         except KeyboardInterrupt:
2289                 sys.exit(u'\nERROR: Interrupted by user')