_ Git - youtube-dl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # Author: Benjamin Johnson
   6 # License: Public domain code
   7 import htmlentitydefs
   8 import httplib
   9 import locale
  10 import math
  11 import netrc
  12 import os
  13 import os.path
  14 import re
  15 import socket
  16 import string
  17 import subprocess
  18 import sys
  19 import time
  20 import urllib
  21 import urllib2
  22
  23 # parse_qs was moved from the cgi module to the urlparse module recently.
  24 try:
  25         from urlparse import parse_qs
  26 except ImportError:
  27         from cgi import parse_qs
  28
  29 std_headers = {
  30         'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6',
  31         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  32         'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
  33         'Accept-Language': 'en-us,en;q=0.5',
  34 }
  35
  36 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  37
  38 def preferredencoding():
  39         """Get preferred encoding.
  40
  41         Returns the best encoding scheme for the system, based on
  42         locale.getpreferredencoding() and some further tweaks.
  43         """
  44         def yield_preferredencoding():
  45                 try:
  46                         pref = locale.getpreferredencoding()
  47                         u'TEST'.encode(pref)
  48                 except:
  49                         pref = 'UTF-8'
  50                 while True:
  51                         yield pref
  52         return yield_preferredencoding().next()
  53
  54 def htmlentity_transform(matchobj):
  55         """Transforms an HTML entity to a Unicode character.
  56
  57         This function receives a match object and is intended to be used with
  58         the re.sub() function.
  59         """
  60         entity = matchobj.group(1)
  61
  62         # Known non-numeric HTML entity
  63         if entity in htmlentitydefs.name2codepoint:
  64                 return unichr(htmlentitydefs.name2codepoint[entity])
  65
  66         # Unicode character
  67         mobj = re.match(ur'(?u)#(x?\d+)', entity)
  68         if mobj is not None:
  69                 numstr = mobj.group(1)
  70                 if numstr.startswith(u'x'):
  71                         base = 16
  72                         numstr = u'0%s' % numstr
  73                 else:
  74                         base = 10
  75                 return unichr(long(numstr, base))
  76
  77         # Unknown entity in name, return its literal representation
  78         return (u'&%s;' % entity)
  79
  80 def sanitize_title(utitle):
  81         """Sanitizes a video title so it could be used as part of a filename."""
  82         utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
  83         return utitle.replace(unicode(os.sep), u'%')
  84
  85 def sanitize_open(filename, open_mode):
  86         """Try to open the given filename, and slightly tweak it if this fails.
  87
  88         Attempts to open the given filename. If this fails, it tries to change
  89         the filename slightly, step by step, until it's either able to open it
  90         or it fails and raises a final exception, like the standard open()
  91         function.
  92
  93         It returns the tuple (stream, definitive_file_name).
  94         """
  95         try:
  96                 stream = open(filename, open_mode)
  97                 return (stream, filename)
  98         except (IOError, OSError), err:
  99                 # In case of error, try to remove win32 forbidden chars
 100                 filename = re.sub(ur'[<>:"\|\?\*]', u'#', filename)
 101
 102                 # An exception here should be caught in the caller
 103                 stream = open(filename, open_mode)
 104                 return (stream, filename)
 105
 106
 107 class DownloadError(Exception):
 108         """Download Error exception.
 109
 110         This exception may be thrown by FileDownloader objects if they are not
 111         configured to continue on errors. They will contain the appropriate
 112         error message.
 113         """
 114         pass
 115
 116 class SameFileError(Exception):
 117         """Same File exception.
 118
 119         This exception will be thrown by FileDownloader objects if they detect
 120         multiple files would have to be downloaded to the same file on disk.
 121         """
 122         pass
 123
 124 class PostProcessingError(Exception):
 125         """Post Processing exception.
 126
 127         This exception may be raised by PostProcessor's .run() method to
 128         indicate an error in the postprocessing task.
 129         """
 130         pass
 131
 132 class UnavailableFormatError(Exception):
 133         """Unavailable Format exception.
 134
 135         This exception will be thrown when a video is requested
 136         in a format that is not available for that video.
 137         """
 138         pass
 139
 140 class ContentTooShortError(Exception):
 141         """Content Too Short exception.
 142
 143         This exception may be raised by FileDownloader objects when a file they
 144         download is too small for what the server announced first, indicating
 145         the connection was probably interrupted.
 146         """
 147         # Both in bytes
 148         downloaded = None
 149         expected = None
 150
 151         def __init__(self, downloaded, expected):
 152                 self.downloaded = downloaded
 153                 self.expected = expected
 154
 155 class FileDownloader(object):
 156         """File Downloader class.
 157
 158         File downloader objects are the ones responsible of downloading the
 159         actual video file and writing it to disk if the user has requested
 160         it, among some other tasks. In most cases there should be one per
 161         program. As, given a video URL, the downloader doesn't know how to
 162         extract all the needed information, task that InfoExtractors do, it
 163         has to pass the URL to one of them.
 164
 165         For this, file downloader objects have a method that allows
 166         InfoExtractors to be registered in a given order. When it is passed
 167         a URL, the file downloader handles it to the first InfoExtractor it
 168         finds that reports being able to handle it. The InfoExtractor extracts
 169         all the information about the video or videos the URL refers to, and
 170         asks the FileDownloader to process the video information, possibly
 171         downloading the video.
 172
 173         File downloaders accept a lot of parameters. In order not to saturate
 174         the object constructor with arguments, it receives a dictionary of
 175         options instead. These options are available through the params
 176         attribute for the InfoExtractors to use. The FileDownloader also
 177         registers itself as the downloader in charge for the InfoExtractors
 178         that are added to it, so this is a "mutual registration".
 179
 180         Available options:
 181
 182         username:       Username for authentication purposes.
 183         password:       Password for authentication purposes.
 184         usenetrc:       Use netrc for authentication instead.
 185         quiet:          Do not print messages to stdout.
 186         forceurl:       Force printing final URL.
 187         forcetitle:     Force printing title.
 188         simulate:       Do not download the video files.
 189         format:         Video format code.
 190         outtmpl:        Template for output names.
 191         ignoreerrors:   Do not stop on download errors.
 192         ratelimit:      Download speed limit, in bytes/sec.
 193         nooverwrites:   Prevent overwriting files.
 194         continuedl:     Try to continue downloads if possible.
 195         """
 196
 197         params = None
 198         _ies = []
 199         _pps = []
 200         _download_retcode = None
 201
 202         def __init__(self, params):
 203                 """Create a FileDownloader object with the given options."""
 204                 self._ies = []
 205                 self._pps = []
 206                 self._download_retcode = 0
 207                 self.params = params
 208
 209         @staticmethod
 210         def pmkdir(filename):
 211                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 212                 components = filename.split(os.sep)
 213                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 214                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 215                 for dir in aggregate:
 216                         if not os.path.exists(dir):
 217                                 os.mkdir(dir)
 218
 219         @staticmethod
 220         def format_bytes(bytes):
 221                 if bytes is None:
 222                         return 'N/A'
 223                 if type(bytes) is str:
 224                         bytes = float(bytes)
 225                 if bytes == 0.0:
 226                         exponent = 0
 227                 else:
 228                         exponent = long(math.log(bytes, 1024.0))
 229                 suffix = 'bkMGTPEZY'[exponent]
 230                 converted = float(bytes) / float(1024**exponent)
 231                 return '%.2f%s' % (converted, suffix)
 232
 233         @staticmethod
 234         def calc_percent(byte_counter, data_len):
 235                 if data_len is None:
 236                         return '---.-%'
 237                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 238
 239         @staticmethod
 240         def calc_eta(start, now, total, current):
 241                 if total is None:
 242                         return '--:--'
 243                 dif = now - start
 244                 if current == 0 or dif < 0.001: # One millisecond
 245                         return '--:--'
 246                 rate = float(current) / dif
 247                 eta = long((float(total) - float(current)) / rate)
 248                 (eta_mins, eta_secs) = divmod(eta, 60)
 249                 if eta_mins > 99:
 250                         return '--:--'
 251                 return '%02d:%02d' % (eta_mins, eta_secs)
 252
 253         @staticmethod
 254         def calc_speed(start, now, bytes):
 255                 dif = now - start
 256                 if bytes == 0 or dif < 0.001: # One millisecond
 257                         return '%10s' % '---b/s'
 258                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 259
 260         @staticmethod
 261         def best_block_size(elapsed_time, bytes):
 262                 new_min = max(bytes / 2.0, 1.0)
 263                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 264                 if elapsed_time < 0.001:
 265                         return long(new_max)
 266                 rate = bytes / elapsed_time
 267                 if rate > new_max:
 268                         return long(new_max)
 269                 if rate < new_min:
 270                         return long(new_min)
 271                 return long(rate)
 272
 273         @staticmethod
 274         def parse_bytes(bytestr):
 275                 """Parse a string indicating a byte quantity into a long integer."""
 276                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 277                 if matchobj is None:
 278                         return None
 279                 number = float(matchobj.group(1))
 280                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 281                 return long(round(number * multiplier))
 282
 283         @staticmethod
 284         def verify_url(url):
 285                 """Verify a URL is valid and data could be downloaded. Return real data URL."""
 286                 request = urllib2.Request(url, None, std_headers)
 287                 data = urllib2.urlopen(request)
 288                 data.read(1)
 289                 url = data.geturl()
 290                 data.close()
 291                 return url
 292
 293         def add_info_extractor(self, ie):
 294                 """Add an InfoExtractor object to the end of the list."""
 295                 self._ies.append(ie)
 296                 ie.set_downloader(self)
 297
 298         def add_post_processor(self, pp):
 299                 """Add a PostProcessor object to the end of the chain."""
 300                 self._pps.append(pp)
 301                 pp.set_downloader(self)
 302
 303         def to_stdout(self, message, skip_eol=False):
 304                 """Print message to stdout if not in quiet mode."""
 305                 if not self.params.get('quiet', False):
 306                         print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
 307                         sys.stdout.flush()
 308
 309         def to_stderr(self, message):
 310                 """Print message to stderr."""
 311                 print >>sys.stderr, message.encode(preferredencoding())
 312
 313         def fixed_template(self):
 314                 """Checks if the output template is fixed."""
 315                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 316
 317         def trouble(self, message=None):
 318                 """Determine action to take when a download problem appears.
 319
 320                 Depending on if the downloader has been configured to ignore
 321                 download errors or not, this method may throw an exception or
 322                 not when errors are found, after printing the message.
 323                 """
 324                 if message is not None:
 325                         self.to_stderr(message)
 326                 if not self.params.get('ignoreerrors', False):
 327                         raise DownloadError(message)
 328                 self._download_retcode = 1
 329
 330         def slow_down(self, start_time, byte_counter):
 331                 """Sleep if the download speed is over the rate limit."""
 332                 rate_limit = self.params.get('ratelimit', None)
 333                 if rate_limit is None or byte_counter == 0:
 334                         return
 335                 now = time.time()
 336                 elapsed = now - start_time
 337                 if elapsed <= 0.0:
 338                         return
 339                 speed = float(byte_counter) / elapsed
 340                 if speed > rate_limit:
 341                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 342
 343         def report_destination(self, filename):
 344                 """Report destination filename."""
 345                 self.to_stdout(u'[download] Destination: %s' % filename)
 346
 347         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 348                 """Report download progress."""
 349                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 350                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 351
 352         def report_resuming_byte(self, resume_len):
 353                 """Report attemtp to resume at given byte."""
 354                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 355
 356         def report_file_already_downloaded(self, file_name):
 357                 """Report file has already been fully downloaded."""
 358                 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 359
 360         def report_unable_to_resume(self):
 361                 """Report it was impossible to resume download."""
 362                 self.to_stdout(u'[download] Unable to resume')
 363
 364         def report_finish(self):
 365                 """Report download finished."""
 366                 self.to_stdout(u'')
 367
 368         def process_info(self, info_dict):
 369                 """Process a single dictionary returned by an InfoExtractor."""
 370                 # Do nothing else if in simulate mode
 371                 if self.params.get('simulate', False):
 372                         # Verify URL if it's an HTTP one
 373                         if info_dict['url'].startswith('http'):
 374                                 try:
 375                                         info_dict['url'] = self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8')
 376                                 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 377                                         raise UnavailableFormatError
 378
 379                         # Forced printings
 380                         if self.params.get('forcetitle', False):
 381                                 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 382                         if self.params.get('forceurl', False):
 383                                 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 384
 385                         return
 386
 387                 try:
 388                         template_dict = dict(info_dict)
 389                         template_dict['epoch'] = unicode(long(time.time()))
 390                         filename = self.params['outtmpl'] % template_dict
 391                 except (ValueError, KeyError), err:
 392                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 393                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 394                         self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
 395                         return
 396
 397                 try:
 398                         self.pmkdir(filename)
 399                 except (OSError, IOError), err:
 400                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 401                         return
 402
 403                 try:
 404                         success = self._do_download(filename, info_dict['url'].encode('utf-8'))
 405                 except (OSError, IOError), err:
 406                         raise UnavailableFormatError
 407                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 408                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 409                         return
 410                 except (ContentTooShortError, ), err:
 411                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 412                         return
 413
 414                 if success:
 415                         try:
 416                                 self.post_process(filename, info_dict)
 417                         except (PostProcessingError), err:
 418                                 self.trouble('ERROR: postprocessing: %s' % str(err))
 419                                 return
 420
 421         def download(self, url_list):
 422                 """Download a given list of URLs."""
 423                 if len(url_list) > 1 and self.fixed_template():
 424                         raise SameFileError(self.params['outtmpl'])
 425
 426                 for url in url_list:
 427                         suitable_found = False
 428                         for ie in self._ies:
 429                                 # Go to next InfoExtractor if not suitable
 430                                 if not ie.suitable(url):
 431                                         continue
 432
 433                                 # Suitable InfoExtractor found
 434                                 suitable_found = True
 435
 436                                 # Extract information from URL and process it
 437                                 ie.extract(url)
 438
 439                                 # Suitable InfoExtractor had been found; go to next URL
 440                                 break
 441
 442                         if not suitable_found:
 443                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 444
 445                 return self._download_retcode
 446
 447         def post_process(self, filename, ie_info):
 448                 """Run the postprocessing chain on the given file."""
 449                 info = dict(ie_info)
 450                 info['filepath'] = filename
 451                 for pp in self._pps:
 452                         info = pp.run(info)
 453                         if info is None:
 454                                 break
 455
 456         def _download_with_rtmpdump(self, filename, url):
 457                 self.report_destination(filename)
 458
 459                 # Check for rtmpdump first
 460                 try:
 461                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 462                 except (OSError, IOError):
 463                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 464                         return False
 465
 466                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 467                 # the connection was interrumpted and resuming appears to be
 468                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 469                 basic_args = ['rtmpdump', '-q', '-r', url, '-o', filename]
 470                 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
 471                 while retval == 2 or retval == 1:
 472                         self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename), skip_eol=True)
 473                         time.sleep(2.0) # This seems to be needed
 474                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 475                 if retval == 0:
 476                         self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
 477                         return True
 478                 else:
 479                         self.trouble('ERROR: rtmpdump exited with code %d' % retval)
 480                         return False
 481
 482         def _do_download(self, filename, url):
 483                 # Attempt to download using rtmpdump
 484                 if url.startswith('rtmp'):
 485                         return self._download_with_rtmpdump(filename, url)
 486
 487                 stream = None
 488                 open_mode = 'wb'
 489                 basic_request = urllib2.Request(url, None, std_headers)
 490                 request = urllib2.Request(url, None, std_headers)
 491
 492                 # Establish possible resume length
 493                 if os.path.isfile(filename):
 494                         resume_len = os.path.getsize(filename)
 495                 else:
 496                         resume_len = 0
 497
 498                 # Request parameters in case of being able to resume
 499                 if self.params.get('continuedl', False) and resume_len != 0:
 500                         self.report_resuming_byte(resume_len)
 501                         request.add_header('Range','bytes=%d-' % resume_len)
 502                         open_mode = 'ab'
 503
 504                 # Establish connection
 505                 try:
 506                         data = urllib2.urlopen(request)
 507                 except (urllib2.HTTPError, ), err:
 508                         if err.code != 416: #  416 is 'Requested range not satisfiable'
 509                                 raise
 510                         # Unable to resume
 511                         data = urllib2.urlopen(basic_request)
 512                         content_length = data.info()['Content-Length']
 513
 514                         if content_length is not None and long(content_length) == resume_len:
 515                                 # Because the file had already been fully downloaded
 516                                 self.report_file_already_downloaded(filename)
 517                                 return True
 518                         else:
 519                                 # Because the server didn't let us
 520                                 self.report_unable_to_resume()
 521                                 open_mode = 'wb'
 522
 523                 data_len = data.info().get('Content-length', None)
 524                 data_len_str = self.format_bytes(data_len)
 525                 byte_counter = 0
 526                 block_size = 1024
 527                 start = time.time()
 528                 while True:
 529                         # Download and write
 530                         before = time.time()
 531                         data_block = data.read(block_size)
 532                         after = time.time()
 533                         data_block_len = len(data_block)
 534                         if data_block_len == 0:
 535                                 break
 536                         byte_counter += data_block_len
 537
 538                         # Open file just in time
 539                         if stream is None:
 540                                 try:
 541                                         (stream, filename) = sanitize_open(filename, open_mode)
 542                                         self.report_destination(filename)
 543                                 except (OSError, IOError), err:
 544                                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 545                                         return False
 546                         stream.write(data_block)
 547                         block_size = self.best_block_size(after - before, data_block_len)
 548
 549                         # Progress message
 550                         percent_str = self.calc_percent(byte_counter, data_len)
 551                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 552                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 553                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 554
 555                         # Apply rate limit
 556                         self.slow_down(start, byte_counter)
 557
 558                 self.report_finish()
 559                 if data_len is not None and str(byte_counter) != data_len:
 560                         raise ContentTooShortError(byte_counter, long(data_len))
 561                 return True
 562
 563 class InfoExtractor(object):
 564         """Information Extractor class.
 565
 566         Information extractors are the classes that, given a URL, extract
 567         information from the video (or videos) the URL refers to. This
 568         information includes the real video URL, the video title and simplified
 569         title, author and others. The information is stored in a dictionary
 570         which is then passed to the FileDownloader. The FileDownloader
 571         processes this information possibly downloading the video to the file
 572         system, among other possible outcomes. The dictionaries must include
 573         the following fields:
 574
 575         id:             Video identifier.
 576         url:            Final video URL.
 577         uploader:       Nickname of the video uploader.
 578         title:          Literal title.
 579         stitle:         Simplified title.
 580         ext:            Video filename extension.
 581
 582         Subclasses of this one should re-define the _real_initialize() and
 583         _real_extract() methods, as well as the suitable() static method.
 584         Probably, they should also be instantiated and added to the main
 585         downloader.
 586         """
 587
 588         _ready = False
 589         _downloader = None
 590
 591         def __init__(self, downloader=None):
 592                 """Constructor. Receives an optional downloader."""
 593                 self._ready = False
 594                 self.set_downloader(downloader)
 595
 596         @staticmethod
 597         def suitable(url):
 598                 """Receives a URL and returns True if suitable for this IE."""
 599                 return False
 600
 601         def initialize(self):
 602                 """Initializes an instance (authentication, etc)."""
 603                 if not self._ready:
 604                         self._real_initialize()
 605                         self._ready = True
 606
 607         def extract(self, url):
 608                 """Extracts URL information and returns it in list of dicts."""
 609                 self.initialize()
 610                 return self._real_extract(url)
 611
 612         def set_downloader(self, downloader):
 613                 """Sets the downloader for this IE."""
 614                 self._downloader = downloader
 615
 616         def _real_initialize(self):
 617                 """Real initialization process. Redefine in subclasses."""
 618                 pass
 619
 620         def _real_extract(self, url):
 621                 """Real extraction process. Redefine in subclasses."""
 622                 pass
 623
 624 class YoutubeIE(InfoExtractor):
 625         """Information extractor for youtube.com."""
 626
 627         _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?[\?#](?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 628         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 629         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 630         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 631         _NETRC_MACHINE = 'youtube'
 632         _available_formats = ['37', '22', '35', '18', '5', '17', '13', None] # listed in order of priority for -b flag
 633         _video_extensions = {
 634                 '13': '3gp',
 635                 '17': 'mp4',
 636                 '18': 'mp4',
 637                 '22': 'mp4',
 638                 '37': 'mp4',
 639         }
 640
 641         @staticmethod
 642         def suitable(url):
 643                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 644
 645         def report_lang(self):
 646                 """Report attempt to set language."""
 647                 self._downloader.to_stdout(u'[youtube] Setting language')
 648
 649         def report_login(self):
 650                 """Report attempt to log in."""
 651                 self._downloader.to_stdout(u'[youtube] Logging in')
 652
 653         def report_age_confirmation(self):
 654                 """Report attempt to confirm age."""
 655                 self._downloader.to_stdout(u'[youtube] Confirming age')
 656
 657         def report_video_info_webpage_download(self, video_id):
 658                 """Report attempt to download video info webpage."""
 659                 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
 660
 661         def report_information_extraction(self, video_id):
 662                 """Report attempt to extract video information."""
 663                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 664
 665         def report_unavailable_format(self, video_id, format):
 666                 """Report extracted video URL."""
 667                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 668
 669         def report_rtmp_download(self):
 670                 """Indicate the download will use the RTMP protocol."""
 671                 self._downloader.to_stdout(u'[youtube] RTMP download detected')
 672
 673         def _real_initialize(self):
 674                 if self._downloader is None:
 675                         return
 676
 677                 username = None
 678                 password = None
 679                 downloader_params = self._downloader.params
 680
 681                 # Attempt to use provided username and password or .netrc data
 682                 if downloader_params.get('username', None) is not None:
 683                         username = downloader_params['username']
 684                         password = downloader_params['password']
 685                 elif downloader_params.get('usenetrc', False):
 686                         try:
 687                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 688                                 if info is not None:
 689                                         username = info[0]
 690                                         password = info[2]
 691                                 else:
 692                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 693                         except (IOError, netrc.NetrcParseError), err:
 694                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 695                                 return
 696
 697                 # Set language
 698                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 699                 try:
 700                         self.report_lang()
 701                         urllib2.urlopen(request).read()
 702                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 703                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 704                         return
 705
 706                 # No authentication to be performed
 707                 if username is None:
 708                         return
 709
 710                 # Log in
 711                 login_form = {
 712                                 'current_form': 'loginForm',
 713                                 'next':         '/',
 714                                 'action_login': 'Log In',
 715                                 'username':     username,
 716                                 'password':     password,
 717                                 }
 718                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 719                 try:
 720                         self.report_login()
 721                         login_results = urllib2.urlopen(request).read()
 722                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 723                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 724                                 return
 725                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 726                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 727                         return
 728
 729                 # Confirm age
 730                 age_form = {
 731                                 'next_url':             '/',
 732                                 'action_confirm':       'Confirm',
 733                                 }
 734                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 735                 try:
 736                         self.report_age_confirmation()
 737                         age_results = urllib2.urlopen(request).read()
 738                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 739                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 740                         return
 741
 742         def _real_extract(self, url):
 743                 # Extract video id from URL
 744                 mobj = re.match(self._VALID_URL, url)
 745                 if mobj is None:
 746                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 747                         return
 748                 video_id = mobj.group(2)
 749
 750                 # Downloader parameters
 751                 best_quality = False
 752                 format_param = None
 753                 quality_index = 0
 754                 if self._downloader is not None:
 755                         params = self._downloader.params
 756                         format_param = params.get('format', None)
 757                         if format_param == '0':
 758                                 format_param = self._available_formats[quality_index]
 759                                 best_quality = True
 760
 761                 while True:
 762                         # Extension
 763                         video_extension = self._video_extensions.get(format_param, 'flv')
 764
 765                         # Get video info
 766                         video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id
 767                         request = urllib2.Request(video_info_url, None, std_headers)
 768                         try:
 769                                 self.report_video_info_webpage_download(video_id)
 770                                 video_info_webpage = urllib2.urlopen(request).read()
 771                                 video_info = parse_qs(video_info_webpage)
 772                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 773                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 774                                 return
 775                         self.report_information_extraction(video_id)
 776
 777                         # "t" param
 778                         if 'token' not in video_info:
 779                                 # Attempt to see if YouTube has issued an error message
 780                                 if 'reason' not in video_info:
 781                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
 782                                         stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
 783                                         stream.write(video_info_webpage)
 784                                         stream.close()
 785                                 else:
 786                                         reason = urllib.unquote_plus(video_info['reason'][0])
 787                                         self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
 788                                 return
 789                         token = urllib.unquote_plus(video_info['token'][0])
 790                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
 791                         if format_param is not None:
 792                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 793
 794                         # Check possible RTMP download
 795                         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 796                                 self.report_rtmp_download()
 797                                 video_real_url = video_info['conn'][0]
 798
 799                         # uploader
 800                         if 'author' not in video_info:
 801                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 802                                 return
 803                         video_uploader = urllib.unquote_plus(video_info['author'][0])
 804
 805                         # title
 806                         if 'title' not in video_info:
 807                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 808                                 return
 809                         video_title = urllib.unquote_plus(video_info['title'][0])
 810                         video_title = video_title.decode('utf-8')
 811                         video_title = sanitize_title(video_title)
 812
 813                         # simplified title
 814                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 815                         simple_title = simple_title.strip(ur'_')
 816
 817                         try:
 818                                 # Process video information
 819                                 self._downloader.process_info({
 820                                         'id':           video_id.decode('utf-8'),
 821                                         'url':          video_real_url.decode('utf-8'),
 822                                         'uploader':     video_uploader.decode('utf-8'),
 823                                         'title':        video_title,
 824                                         'stitle':       simple_title,
 825                                         'ext':          video_extension.decode('utf-8'),
 826                                 })
 827
 828                                 return
 829
 830                         except UnavailableFormatError, err:
 831                                 if best_quality:
 832                                         if quality_index == len(self._available_formats) - 1:
 833                                                 # I don't ever expect this to happen
 834                                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 835                                                 return
 836                                         else:
 837                                                 self.report_unavailable_format(video_id, format_param)
 838                                                 quality_index += 1
 839                                                 format_param = self._available_formats[quality_index]
 840                                                 continue
 841                                 else:
 842                                         self._downloader.trouble('ERROR: format not available for video')
 843                                         return
 844
 845
 846 class MetacafeIE(InfoExtractor):
 847         """Information Extractor for metacafe.com."""
 848
 849         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 850         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 851         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 852         _youtube_ie = None
 853
 854         def __init__(self, youtube_ie, downloader=None):
 855                 InfoExtractor.__init__(self, downloader)
 856                 self._youtube_ie = youtube_ie
 857
 858         @staticmethod
 859         def suitable(url):
 860                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 861
 862         def report_disclaimer(self):
 863                 """Report disclaimer retrieval."""
 864                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 865
 866         def report_age_confirmation(self):
 867                 """Report attempt to confirm age."""
 868                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 869
 870         def report_download_webpage(self, video_id):
 871                 """Report webpage download."""
 872                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
 873
 874         def report_extraction(self, video_id):
 875                 """Report information extraction."""
 876                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
 877
 878         def _real_initialize(self):
 879                 # Retrieve disclaimer
 880                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
 881                 try:
 882                         self.report_disclaimer()
 883                         disclaimer = urllib2.urlopen(request).read()
 884                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 885                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 886                         return
 887
 888                 # Confirm age
 889                 disclaimer_form = {
 890                         'filters': '0',
 891                         'submit': "Continue - I'm over 18",
 892                         }
 893                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
 894                 try:
 895                         self.report_age_confirmation()
 896                         disclaimer = urllib2.urlopen(request).read()
 897                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 898                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 899                         return
 900
 901         def _real_extract(self, url):
 902                 # Extract id and simplified title from URL
 903                 mobj = re.match(self._VALID_URL, url)
 904                 if mobj is None:
 905                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 906                         return
 907
 908                 video_id = mobj.group(1)
 909
 910                 # Check if video comes from YouTube
 911                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 912                 if mobj2 is not None:
 913                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 914                         return
 915
 916                 simple_title = mobj.group(2).decode('utf-8')
 917                 video_extension = 'flv'
 918
 919                 # Retrieve video webpage to extract further information
 920                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 921                 try:
 922                         self.report_download_webpage(video_id)
 923                         webpage = urllib2.urlopen(request).read()
 924                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 925                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 926                         return
 927
 928                 # Extract URL, uploader and title from webpage
 929                 self.report_extraction(video_id)
 930                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 931                 if mobj is None:
 932                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 933                         return
 934                 mediaURL = urllib.unquote(mobj.group(1))
 935
 936                 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 937                 #if mobj is None:
 938                 #       self._downloader.trouble(u'ERROR: unable to extract gdaKey')
 939                 #       return
 940                 #gdaKey = mobj.group(1)
 941                 #
 942                 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 943
 944                 video_url = mediaURL
 945
 946                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 947                 if mobj is None:
 948                         self._downloader.trouble(u'ERROR: unable to extract title')
 949                         return
 950                 video_title = mobj.group(1).decode('utf-8')
 951                 video_title = sanitize_title(video_title)
 952
 953                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
 954                 if mobj is None:
 955                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 956                         return
 957                 video_uploader = mobj.group(1)
 958
 959                 try:
 960                         # Process video information
 961                         self._downloader.process_info({
 962                                 'id':           video_id.decode('utf-8'),
 963                                 'url':          video_url.decode('utf-8'),
 964                                 'uploader':     video_uploader.decode('utf-8'),
 965                                 'title':        video_title,
 966                                 'stitle':       simple_title,
 967                                 'ext':          video_extension.decode('utf-8'),
 968                         })
 969                 except UnavailableFormatError:
 970                         self._downloader.trouble(u'ERROR: format not available for video')
 971
 972
 973 class GoogleIE(InfoExtractor):
 974         """Information extractor for video.google.com."""
 975
 976         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
 977
 978         def __init__(self, downloader=None):
 979                 InfoExtractor.__init__(self, downloader)
 980
 981         @staticmethod
 982         def suitable(url):
 983                 return (re.match(GoogleIE._VALID_URL, url) is not None)
 984
 985         def report_download_webpage(self, video_id):
 986                 """Report webpage download."""
 987                 self._downloader.to_stdout(u'[video.google] %s: Downloading webpage' % video_id)
 988
 989         def report_extraction(self, video_id):
 990                 """Report information extraction."""
 991                 self._downloader.to_stdout(u'[video.google] %s: Extracting information' % video_id)
 992
 993         def _real_initialize(self):
 994                 return
 995
 996         def _real_extract(self, url):
 997                 # Extract id from URL
 998                 mobj = re.match(self._VALID_URL, url)
 999                 if mobj is None:
1000                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1001                         return
1002
1003                 video_id = mobj.group(1)
1004
1005                 video_extension = 'mp4'
1006
1007                 # Retrieve video webpage to extract further information
1008                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1009                 try:
1010                         self.report_download_webpage(video_id)
1011                         webpage = urllib2.urlopen(request).read()
1012                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1013                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1014                         return
1015
1016                 # Extract URL, uploader, and title from webpage
1017                 self.report_extraction(video_id)
1018                 mobj = re.search(r"download_url:'([^']+)'", webpage)
1019                 if mobj is None:
1020                         video_extension = 'flv'
1021                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1022                 if mobj is None:
1023                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1024                         return
1025                 mediaURL = urllib.unquote(mobj.group(1))
1026                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1027                 mediaURL = mediaURL.replace('\\x26', '\x26')
1028
1029                 video_url = mediaURL
1030
1031                 mobj = re.search(r'<title>(.*)</title>', webpage)
1032                 if mobj is None:
1033                         self._downloader.trouble(u'ERROR: unable to extract title')
1034                         return
1035                 video_title = mobj.group(1).decode('utf-8')
1036                 video_title = sanitize_title(video_title)
1037
1038                 # Google Video doesn't show uploader nicknames?
1039                 video_uploader = 'NA'
1040
1041                 try:
1042                         # Process video information
1043                         self._downloader.process_info({
1044                                 'id':           video_id.decode('utf-8'),
1045                                 'url':          video_url.decode('utf-8'),
1046                                 'uploader':     video_uploader.decode('utf-8'),
1047                                 'title':        video_title,
1048                                 'stitle':       video_title,
1049                                 'ext':          video_extension.decode('utf-8'),
1050                         })
1051                 except UnavailableFormatError:
1052                         self._downloader.trouble(u'ERROR: format not available for video')
1053
1054
1055 class PhotobucketIE(InfoExtractor):
1056         """Information extractor for photobucket.com."""
1057
1058         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1059
1060         def __init__(self, downloader=None):
1061                 InfoExtractor.__init__(self, downloader)
1062
1063         @staticmethod
1064         def suitable(url):
1065                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1066
1067         def report_download_webpage(self, video_id):
1068                 """Report webpage download."""
1069                 self._downloader.to_stdout(u'[photobucket] %s: Downloading webpage' % video_id)
1070
1071         def report_extraction(self, video_id):
1072                 """Report information extraction."""
1073                 self._downloader.to_stdout(u'[photobucket] %s: Extracting information' % video_id)
1074
1075         def _real_initialize(self):
1076                 return
1077
1078         def _real_extract(self, url):
1079                 # Extract id from URL
1080                 mobj = re.match(self._VALID_URL, url)
1081                 if mobj is None:
1082                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1083                         return
1084
1085                 video_id = mobj.group(1)
1086
1087                 video_extension = 'flv'
1088
1089                 # Retrieve video webpage to extract further information
1090                 request = urllib2.Request(url)
1091                 try:
1092                         self.report_download_webpage(video_id)
1093                         webpage = urllib2.urlopen(request).read()
1094                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1095                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1096                         return
1097
1098                 # Extract URL, uploader, and title from webpage
1099                 self.report_extraction(video_id)
1100                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1101                 if mobj is None:
1102                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1103                         return
1104                 mediaURL = urllib.unquote(mobj.group(1))
1105
1106                 video_url = mediaURL
1107
1108                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1109                 if mobj is None:
1110                         self._downloader.trouble(u'ERROR: unable to extract title')
1111                         return
1112                 video_title = mobj.group(1).decode('utf-8')
1113                 video_title = sanitize_title(video_title)
1114
1115                 video_uploader = mobj.group(2).decode('utf-8')
1116
1117                 try:
1118                         # Process video information
1119                         self._downloader.process_info({
1120                                 'id':           video_id.decode('utf-8'),
1121                                 'url':          video_url.decode('utf-8'),
1122                                 'uploader':     video_uploader,
1123                                 'title':        video_title,
1124                                 'stitle':       video_title,
1125                                 'ext':          video_extension.decode('utf-8'),
1126                         })
1127                 except UnavailableFormatError:
1128                         self._downloader.trouble(u'ERROR: format not available for video')
1129
1130
1131 class GenericIE(InfoExtractor):
1132         """Generic last-resort information extractor."""
1133
1134         def __init__(self, downloader=None):
1135                 InfoExtractor.__init__(self, downloader)
1136
1137         @staticmethod
1138         def suitable(url):
1139                 return True
1140
1141         def report_download_webpage(self, video_id):
1142                 """Report webpage download."""
1143                 self._downloader.to_stdout(u'WARNING: Falling back on generic information extractor.')
1144                 self._downloader.to_stdout(u'[generic] %s: Downloading webpage' % video_id)
1145
1146         def report_extraction(self, video_id):
1147                 """Report information extraction."""
1148                 self._downloader.to_stdout(u'[generic] %s: Extracting information' % video_id)
1149
1150         def _real_initialize(self):
1151                 return
1152
1153         def _real_extract(self, url):
1154                 video_id = url.split('/')[-1]
1155                 request = urllib2.Request(url)
1156                 try:
1157                         self.report_download_webpage(video_id)
1158                         webpage = urllib2.urlopen(request).read()
1159                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1160                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1161                         return
1162                 except ValueError, err:
1163                         # since this is the last-resort InfoExtractor, if
1164                         # this error is thrown, it'll be thrown here
1165                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1166                         return
1167
1168                 # Start with something easy: JW Player in SWFObject
1169                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1170                 if mobj is None:
1171                         # Broaden the search a little bit
1172                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1173                 if mobj is None:
1174                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1175                         return
1176
1177                 # It's possible that one of the regexes
1178                 # matched, but returned an empty group:
1179                 if mobj.group(1) is None:
1180                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1181                         return
1182
1183                 video_url = urllib.unquote(mobj.group(1))
1184                 video_id  = os.path.basename(video_url)
1185
1186                 # here's a fun little line of code for you:
1187                 video_extension = os.path.splitext(video_id)[1][1:]
1188                 video_id        = os.path.splitext(video_id)[0]
1189
1190                 # it's tempting to parse this further, but you would
1191                 # have to take into account all the variations like
1192                 #   Video Title - Site Name
1193                 #   Site Name | Video Title
1194                 #   Video Title - Tagline | Site Name
1195                 # and so on and so forth; it's just not practical
1196                 mobj = re.search(r'<title>(.*)</title>', webpage)
1197                 if mobj is None:
1198                         self._downloader.trouble(u'ERROR: unable to extract title')
1199                         return
1200                 video_title = mobj.group(1).decode('utf-8')
1201                 video_title = sanitize_title(video_title)
1202
1203                 # video uploader is domain name
1204                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1205                 if mobj is None:
1206                         self._downloader.trouble(u'ERROR: unable to extract title')
1207                         return
1208                 video_uploader = mobj.group(1).decode('utf-8')
1209
1210                 try:
1211                         # Process video information
1212                         self._downloader.process_info({
1213                                 'id':           video_id.decode('utf-8'),
1214                                 'url':          video_url.decode('utf-8'),
1215                                 'uploader':     video_uploader,
1216                                 'title':        video_title,
1217                                 'stitle':       video_title,
1218                                 'ext':          video_extension.decode('utf-8'),
1219                         })
1220                 except UnavailableFormatError:
1221                         self._downloader.trouble(u'ERROR: format not available for video')
1222
1223
1224 class YoutubeSearchIE(InfoExtractor):
1225         """Information Extractor for YouTube search queries."""
1226         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1227         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1228         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1229         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1230         _youtube_ie = None
1231         _max_youtube_results = 1000
1232
1233         def __init__(self, youtube_ie, downloader=None):
1234                 InfoExtractor.__init__(self, downloader)
1235                 self._youtube_ie = youtube_ie
1236
1237         @staticmethod
1238         def suitable(url):
1239                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1240
1241         def report_download_page(self, query, pagenum):
1242                 """Report attempt to download playlist page with given number."""
1243                 query = query.decode(preferredencoding())
1244                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1245
1246         def _real_initialize(self):
1247                 self._youtube_ie.initialize()
1248
1249         def _real_extract(self, query):
1250                 mobj = re.match(self._VALID_QUERY, query)
1251                 if mobj is None:
1252                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1253                         return
1254
1255                 prefix, query = query.split(':')
1256                 prefix = prefix[8:]
1257                 query  = query.encode('utf-8')
1258                 if prefix == '':
1259                         self._download_n_results(query, 1)
1260                         return
1261                 elif prefix == 'all':
1262                         self._download_n_results(query, self._max_youtube_results)
1263                         return
1264                 else:
1265                         try:
1266                                 n = long(prefix)
1267                                 if n <= 0:
1268                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1269                                         return
1270                                 elif n > self._max_youtube_results:
1271                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
1272                                         n = self._max_youtube_results
1273                                 self._download_n_results(query, n)
1274                                 return
1275                         except ValueError: # parsing prefix as integer fails
1276                                 self._download_n_results(query, 1)
1277                                 return
1278
1279         def _download_n_results(self, query, n):
1280                 """Downloads a specified number of results for a query"""
1281
1282                 video_ids = []
1283                 already_seen = set()
1284                 pagenum = 1
1285
1286                 while True:
1287                         self.report_download_page(query, pagenum)
1288                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1289                         request = urllib2.Request(result_url, None, std_headers)
1290                         try:
1291                                 page = urllib2.urlopen(request).read()
1292                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1293                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1294                                 return
1295
1296                         # Extract video identifiers
1297                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1298                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1299                                 if video_id not in already_seen:
1300                                         video_ids.append(video_id)
1301                                         already_seen.add(video_id)
1302                                         if len(video_ids) == n:
1303                                                 # Specified n videos reached
1304                                                 for id in video_ids:
1305                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1306                                                 return
1307
1308                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1309                                 for id in video_ids:
1310                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1311                                 return
1312
1313                         pagenum = pagenum + 1
1314
1315 class YoutubePlaylistIE(InfoExtractor):
1316         """Information Extractor for YouTube playlists."""
1317
1318         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:view_play_list|my_playlists)\?.*?p=([^&]+).*'
1319         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1320         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1321         _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
1322         _youtube_ie = None
1323
1324         def __init__(self, youtube_ie, downloader=None):
1325                 InfoExtractor.__init__(self, downloader)
1326                 self._youtube_ie = youtube_ie
1327
1328         @staticmethod
1329         def suitable(url):
1330                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1331
1332         def report_download_page(self, playlist_id, pagenum):
1333                 """Report attempt to download playlist page with given number."""
1334                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1335
1336         def _real_initialize(self):
1337                 self._youtube_ie.initialize()
1338
1339         def _real_extract(self, url):
1340                 # Extract playlist id
1341                 mobj = re.match(self._VALID_URL, url)
1342                 if mobj is None:
1343                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1344                         return
1345
1346                 # Download playlist pages
1347                 playlist_id = mobj.group(1)
1348                 video_ids = []
1349                 pagenum = 1
1350
1351                 while True:
1352                         self.report_download_page(playlist_id, pagenum)
1353                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1354                         try:
1355                                 page = urllib2.urlopen(request).read()
1356                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1357                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1358                                 return
1359
1360                         # Extract video identifiers
1361                         ids_in_page = []
1362                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1363                                 if mobj.group(1) not in ids_in_page:
1364                                         ids_in_page.append(mobj.group(1))
1365                         video_ids.extend(ids_in_page)
1366
1367                         if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page:
1368                                 break
1369                         pagenum = pagenum + 1
1370
1371                 for id in video_ids:
1372                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1373                 return
1374
1375 class YoutubeUserIE(InfoExtractor):
1376         """Information Extractor for YouTube users."""
1377
1378         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
1379         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1380         _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
1381         _youtube_ie = None
1382
1383         def __init__(self, youtube_ie, downloader=None):
1384                 InfoExtractor.__init__(self, downloader)
1385                 self._youtube_ie = youtube_ie
1386
1387         @staticmethod
1388         def suitable(url):
1389                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
1390
1391         def report_download_page(self, username):
1392                 """Report attempt to download user page."""
1393                 self._downloader.to_stdout(u'[youtube] user %s: Downloading page ' % (username))
1394
1395         def _real_initialize(self):
1396                 self._youtube_ie.initialize()
1397
1398         def _real_extract(self, url):
1399                 # Extract username
1400                 mobj = re.match(self._VALID_URL, url)
1401                 if mobj is None:
1402                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1403                         return
1404
1405                 # Download user page
1406                 username = mobj.group(1)
1407                 video_ids = []
1408                 pagenum = 1
1409
1410                 self.report_download_page(username)
1411                 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
1412                 try:
1413                         page = urllib2.urlopen(request).read()
1414                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1415                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1416                         return
1417
1418                 # Extract video identifiers
1419                 ids_in_page = []
1420
1421                 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1422                         if mobj.group(1) not in ids_in_page:
1423                                 ids_in_page.append(mobj.group(1))
1424                 video_ids.extend(ids_in_page)
1425
1426                 for id in video_ids:
1427                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1428                 return
1429
1430 class PostProcessor(object):
1431         """Post Processor class.
1432
1433         PostProcessor objects can be added to downloaders with their
1434         add_post_processor() method. When the downloader has finished a
1435         successful download, it will take its internal chain of PostProcessors
1436         and start calling the run() method on each one of them, first with
1437         an initial argument and then with the returned value of the previous
1438         PostProcessor.
1439
1440         The chain will be stopped if one of them ever returns None or the end
1441         of the chain is reached.
1442
1443         PostProcessor objects follow a "mutual registration" process similar
1444         to InfoExtractor objects.
1445         """
1446
1447         _downloader = None
1448
1449         def __init__(self, downloader=None):
1450                 self._downloader = downloader
1451
1452         def set_downloader(self, downloader):
1453                 """Sets the downloader for this PP."""
1454                 self._downloader = downloader
1455
1456         def run(self, information):
1457                 """Run the PostProcessor.
1458
1459                 The "information" argument is a dictionary like the ones
1460                 composed by InfoExtractors. The only difference is that this
1461                 one has an extra field called "filepath" that points to the
1462                 downloaded file.
1463
1464                 When this method returns None, the postprocessing chain is
1465                 stopped. However, this method may return an information
1466                 dictionary that will be passed to the next postprocessing
1467                 object in the chain. It can be the one it received after
1468                 changing some fields.
1469
1470                 In addition, this method may raise a PostProcessingError
1471                 exception that will be taken into account by the downloader
1472                 it was called from.
1473                 """
1474                 return information # by default, do nothing
1475
1476 ### MAIN PROGRAM ###
1477 if __name__ == '__main__':
1478         try:
1479                 # Modules needed only when running the main program
1480                 import getpass
1481                 import optparse
1482
1483                 # Function to update the program file with the latest version from bitbucket.org
1484                 def update_self(downloader, filename):
1485                         # Note: downloader only used for options
1486                         if not os.access (filename, os.W_OK):
1487                                 sys.exit('ERROR: no write permissions on %s' % filename)
1488
1489                         downloader.to_stdout('Updating to latest stable version...')
1490                         latest_url = 'http://bitbucket.org/rg3/youtube-dl/raw/tip/LATEST_VERSION'
1491                         latest_version = urllib.urlopen(latest_url).read().strip()
1492                         prog_url = 'http://bitbucket.org/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
1493                         newcontent = urllib.urlopen(prog_url).read()
1494                         stream = open(filename, 'w')
1495                         stream.write(newcontent)
1496                         stream.close()
1497                         downloader.to_stdout('Updated to version %s' % latest_version)
1498
1499                 # General configuration
1500                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1501                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1502                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1503
1504                 # Parse command line
1505                 parser = optparse.OptionParser(
1506                         usage='Usage: %prog [options] url...',
1507                         version='2010.02.13',
1508                         conflict_handler='resolve',
1509                 )
1510
1511                 parser.add_option('-h', '--help',
1512                                 action='help', help='print this help text and exit')
1513                 parser.add_option('-v', '--version',
1514                                 action='version', help='print program version and exit')
1515                 parser.add_option('-U', '--update',
1516                                 action='store_true', dest='update_self', help='update this program to latest stable version')
1517                 parser.add_option('-i', '--ignore-errors',
1518                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1519                 parser.add_option('-r', '--rate-limit',
1520                                 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1521
1522                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1523                 authentication.add_option('-u', '--username',
1524                                 dest='username', metavar='UN', help='account username')
1525                 authentication.add_option('-p', '--password',
1526                                 dest='password', metavar='PW', help='account password')
1527                 authentication.add_option('-n', '--netrc',
1528                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1529                 parser.add_option_group(authentication)
1530
1531                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1532                 video_format.add_option('-f', '--format',
1533                                 action='store', dest='format', metavar='FMT', help='video format code')
1534                 video_format.add_option('-b', '--best-quality',
1535                                 action='store_const', dest='format', help='download the best quality video possible', const='0')
1536                 video_format.add_option('-m', '--mobile-version',
1537                                 action='store_const', dest='format', help='alias for -f 17', const='17')
1538                 video_format.add_option('-d', '--high-def',
1539                                 action='store_const', dest='format', help='alias for -f 22', const='22')
1540                 parser.add_option_group(video_format)
1541
1542                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1543                 verbosity.add_option('-q', '--quiet',
1544                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1545                 verbosity.add_option('-s', '--simulate',
1546                                 action='store_true', dest='simulate', help='do not download video', default=False)
1547                 verbosity.add_option('-g', '--get-url',
1548                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1549                 verbosity.add_option('-e', '--get-title',
1550                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1551                 parser.add_option_group(verbosity)
1552
1553                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1554                 filesystem.add_option('-t', '--title',
1555                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
1556                 filesystem.add_option('-l', '--literal',
1557                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1558                 filesystem.add_option('-o', '--output',
1559                                 dest='outtmpl', metavar='TPL', help='output filename template')
1560                 filesystem.add_option('-a', '--batch-file',
1561                                 dest='batchfile', metavar='F', help='file containing URLs to download')
1562                 filesystem.add_option('-w', '--no-overwrites',
1563                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1564                 filesystem.add_option('-c', '--continue',
1565                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1566                 parser.add_option_group(filesystem)
1567
1568                 (opts, args) = parser.parse_args()
1569
1570                 # Batch file verification
1571                 batchurls = []
1572                 if opts.batchfile is not None:
1573                         try:
1574                                 batchurls = open(opts.batchfile, 'r').readlines()
1575                                 batchurls = [x.strip() for x in batchurls]
1576                                 batchurls = [x for x in batchurls if len(x) > 0]
1577                         except IOError:
1578                                 sys.exit(u'ERROR: batch file could not be read')
1579                 all_urls = batchurls + args
1580
1581                 # Make sure all URLs are in our preferred encoding
1582                 for i in range(0, len(all_urls)):
1583                         all_urls[i] = unicode(all_urls[i], preferredencoding())
1584
1585                 # Conflicting, missing and erroneous options
1586                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1587                         parser.error(u'using .netrc conflicts with giving username/password')
1588                 if opts.password is not None and opts.username is None:
1589                         parser.error(u'account username missing')
1590                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1591                         parser.error(u'using output template conflicts with using title or literal title')
1592                 if opts.usetitle and opts.useliteral:
1593                         parser.error(u'using title conflicts with using literal title')
1594                 if opts.username is not None and opts.password is None:
1595                         opts.password = getpass.getpass(u'Type account password and press return:')
1596                 if opts.ratelimit is not None:
1597                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1598                         if numeric_limit is None:
1599                                 parser.error(u'invalid rate limit specified')
1600                         opts.ratelimit = numeric_limit
1601
1602                 # Information extractors
1603                 youtube_ie = YoutubeIE()
1604                 metacafe_ie = MetacafeIE(youtube_ie)
1605                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1606                 youtube_user_ie = YoutubeUserIE(youtube_ie)
1607                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1608                 google_ie = GoogleIE()
1609                 photobucket_ie = PhotobucketIE()
1610                 generic_ie = GenericIE()
1611
1612                 # File downloader
1613                 fd = FileDownloader({
1614                         'usenetrc': opts.usenetrc,
1615                         'username': opts.username,
1616                         'password': opts.password,
1617                         'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1618                         'forceurl': opts.geturl,
1619                         'forcetitle': opts.gettitle,
1620                         'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1621                         'format': opts.format,
1622                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
1623                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1624                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1625                                 or u'%(id)s.%(ext)s'),
1626                         'ignoreerrors': opts.ignoreerrors,
1627                         'ratelimit': opts.ratelimit,
1628                         'nooverwrites': opts.nooverwrites,
1629                         'continuedl': opts.continue_dl,
1630                         })
1631                 fd.add_info_extractor(youtube_search_ie)
1632                 fd.add_info_extractor(youtube_pl_ie)
1633                 fd.add_info_extractor(youtube_user_ie)
1634                 fd.add_info_extractor(metacafe_ie)
1635                 fd.add_info_extractor(youtube_ie)
1636                 fd.add_info_extractor(google_ie)
1637                 fd.add_info_extractor(photobucket_ie)
1638
1639                 # This must come last since it's the
1640                 # fallback if none of the others work
1641                 fd.add_info_extractor(generic_ie)
1642
1643                 # Update version
1644                 if opts.update_self:
1645                         update_self(fd, sys.argv[0])
1646
1647                 # Maybe do nothing
1648                 if len(all_urls) < 1:
1649                         if not opts.update_self:
1650                                 parser.error(u'you must provide at least one URL')
1651                         else:
1652                                 sys.exit()
1653                 retcode = fd.download(all_urls)
1654                 sys.exit(retcode)
1655
1656         except DownloadError:
1657                 sys.exit(1)
1658         except SameFileError:
1659                 sys.exit(u'ERROR: fixed output name but more than one file to download')
1660         except KeyboardInterrupt:
1661                 sys.exit(u'\nERROR: Interrupted by user')