_ Git - youtube-dl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # Author: Benjamin Johnson
   6 # License: Public domain code
   7 import htmlentitydefs
   8 import httplib
   9 import locale
  10 import math
  11 import netrc
  12 import os
  13 import os.path
  14 import re
  15 import socket
  16 import string
  17 import subprocess
  18 import sys
  19 import time
  20 import urllib
  21 import urllib2
  22
  23 # parse_qs was moved from the cgi module to the urlparse module recently.
  24 try:
  25         from urlparse import parse_qs
  26 except ImportError:
  27         from cgi import parse_qs
  28
  29 std_headers = {
  30         'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2',
  31         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  32         'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
  33         'Accept-Language': 'en-us,en;q=0.5',
  34 }
  35
  36 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  37
  38 def preferredencoding():
  39         """Get preferred encoding.
  40
  41         Returns the best encoding scheme for the system, based on
  42         locale.getpreferredencoding() and some further tweaks.
  43         """
  44         def yield_preferredencoding():
  45                 try:
  46                         pref = locale.getpreferredencoding()
  47                         u'TEST'.encode(pref)
  48                 except:
  49                         pref = 'UTF-8'
  50                 while True:
  51                         yield pref
  52         return yield_preferredencoding().next()
  53
  54 class DownloadError(Exception):
  55         """Download Error exception.
  56
  57         This exception may be thrown by FileDownloader objects if they are not
  58         configured to continue on errors. They will contain the appropriate
  59         error message.
  60         """
  61         pass
  62
  63 class SameFileError(Exception):
  64         """Same File exception.
  65
  66         This exception will be thrown by FileDownloader objects if they detect
  67         multiple files would have to be downloaded to the same file on disk.
  68         """
  69         pass
  70
  71 class PostProcessingError(Exception):
  72         """Post Processing exception.
  73
  74         This exception may be raised by PostProcessor's .run() method to
  75         indicate an error in the postprocessing task.
  76         """
  77         pass
  78
  79 class UnavailableFormatError(Exception):
  80         """Unavailable Format exception.
  81
  82         This exception will be thrown when a video is requested
  83         in a format that is not available for that video.
  84         """
  85         pass
  86
  87 class ContentTooShortError(Exception):
  88         """Content Too Short exception.
  89
  90         This exception may be raised by FileDownloader objects when a file they
  91         download is too small for what the server announced first, indicating
  92         the connection was probably interrupted.
  93         """
  94         # Both in bytes
  95         downloaded = None
  96         expected = None
  97
  98         def __init__(self, downloaded, expected):
  99                 self.downloaded = downloaded
 100                 self.expected = expected
 101
 102 class FileDownloader(object):
 103         """File Downloader class.
 104
 105         File downloader objects are the ones responsible of downloading the
 106         actual video file and writing it to disk if the user has requested
 107         it, among some other tasks. In most cases there should be one per
 108         program. As, given a video URL, the downloader doesn't know how to
 109         extract all the needed information, task that InfoExtractors do, it
 110         has to pass the URL to one of them.
 111
 112         For this, file downloader objects have a method that allows
 113         InfoExtractors to be registered in a given order. When it is passed
 114         a URL, the file downloader handles it to the first InfoExtractor it
 115         finds that reports being able to handle it. The InfoExtractor extracts
 116         all the information about the video or videos the URL refers to, and
 117         asks the FileDownloader to process the video information, possibly
 118         downloading the video.
 119
 120         File downloaders accept a lot of parameters. In order not to saturate
 121         the object constructor with arguments, it receives a dictionary of
 122         options instead. These options are available through the params
 123         attribute for the InfoExtractors to use. The FileDownloader also
 124         registers itself as the downloader in charge for the InfoExtractors
 125         that are added to it, so this is a "mutual registration".
 126
 127         Available options:
 128
 129         username:       Username for authentication purposes.
 130         password:       Password for authentication purposes.
 131         usenetrc:       Use netrc for authentication instead.
 132         quiet:          Do not print messages to stdout.
 133         forceurl:       Force printing final URL.
 134         forcetitle:     Force printing title.
 135         simulate:       Do not download the video files.
 136         format:         Video format code.
 137         outtmpl:        Template for output names.
 138         ignoreerrors:   Do not stop on download errors.
 139         ratelimit:      Download speed limit, in bytes/sec.
 140         nooverwrites:   Prevent overwriting files.
 141         continuedl:     Try to continue downloads if possible.
 142         """
 143
 144         params = None
 145         _ies = []
 146         _pps = []
 147         _download_retcode = None
 148
 149         def __init__(self, params):
 150                 """Create a FileDownloader object with the given options."""
 151                 self._ies = []
 152                 self._pps = []
 153                 self._download_retcode = 0
 154                 self.params = params
 155
 156         @staticmethod
 157         def pmkdir(filename):
 158                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 159                 components = filename.split(os.sep)
 160                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 161                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 162                 for dir in aggregate:
 163                         if not os.path.exists(dir):
 164                                 os.mkdir(dir)
 165
 166         @staticmethod
 167         def format_bytes(bytes):
 168                 if bytes is None:
 169                         return 'N/A'
 170                 if type(bytes) is str:
 171                         bytes = float(bytes)
 172                 if bytes == 0.0:
 173                         exponent = 0
 174                 else:
 175                         exponent = long(math.log(bytes, 1024.0))
 176                 suffix = 'bkMGTPEZY'[exponent]
 177                 converted = float(bytes) / float(1024**exponent)
 178                 return '%.2f%s' % (converted, suffix)
 179
 180         @staticmethod
 181         def calc_percent(byte_counter, data_len):
 182                 if data_len is None:
 183                         return '---.-%'
 184                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 185
 186         @staticmethod
 187         def calc_eta(start, now, total, current):
 188                 if total is None:
 189                         return '--:--'
 190                 dif = now - start
 191                 if current == 0 or dif < 0.001: # One millisecond
 192                         return '--:--'
 193                 rate = float(current) / dif
 194                 eta = long((float(total) - float(current)) / rate)
 195                 (eta_mins, eta_secs) = divmod(eta, 60)
 196                 if eta_mins > 99:
 197                         return '--:--'
 198                 return '%02d:%02d' % (eta_mins, eta_secs)
 199
 200         @staticmethod
 201         def calc_speed(start, now, bytes):
 202                 dif = now - start
 203                 if bytes == 0 or dif < 0.001: # One millisecond
 204                         return '%10s' % '---b/s'
 205                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 206
 207         @staticmethod
 208         def best_block_size(elapsed_time, bytes):
 209                 new_min = max(bytes / 2.0, 1.0)
 210                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 211                 if elapsed_time < 0.001:
 212                         return long(new_max)
 213                 rate = bytes / elapsed_time
 214                 if rate > new_max:
 215                         return long(new_max)
 216                 if rate < new_min:
 217                         return long(new_min)
 218                 return long(rate)
 219
 220         @staticmethod
 221         def parse_bytes(bytestr):
 222                 """Parse a string indicating a byte quantity into a long integer."""
 223                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 224                 if matchobj is None:
 225                         return None
 226                 number = float(matchobj.group(1))
 227                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 228                 return long(round(number * multiplier))
 229
 230         @staticmethod
 231         def verify_url(url):
 232                 """Verify a URL is valid and data could be downloaded. Return real data URL."""
 233                 request = urllib2.Request(url, None, std_headers)
 234                 data = urllib2.urlopen(request)
 235                 data.read(1)
 236                 url = data.geturl()
 237                 data.close()
 238                 return url
 239
 240         def add_info_extractor(self, ie):
 241                 """Add an InfoExtractor object to the end of the list."""
 242                 self._ies.append(ie)
 243                 ie.set_downloader(self)
 244
 245         def add_post_processor(self, pp):
 246                 """Add a PostProcessor object to the end of the chain."""
 247                 self._pps.append(pp)
 248                 pp.set_downloader(self)
 249
 250         def to_stdout(self, message, skip_eol=False):
 251                 """Print message to stdout if not in quiet mode."""
 252                 if not self.params.get('quiet', False):
 253                         print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
 254                         sys.stdout.flush()
 255
 256         def to_stderr(self, message):
 257                 """Print message to stderr."""
 258                 print >>sys.stderr, message.encode(preferredencoding())
 259
 260         def fixed_template(self):
 261                 """Checks if the output template is fixed."""
 262                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 263
 264         def trouble(self, message=None):
 265                 """Determine action to take when a download problem appears.
 266
 267                 Depending on if the downloader has been configured to ignore
 268                 download errors or not, this method may throw an exception or
 269                 not when errors are found, after printing the message.
 270                 """
 271                 if message is not None:
 272                         self.to_stderr(message)
 273                 if not self.params.get('ignoreerrors', False):
 274                         raise DownloadError(message)
 275                 self._download_retcode = 1
 276
 277         def slow_down(self, start_time, byte_counter):
 278                 """Sleep if the download speed is over the rate limit."""
 279                 rate_limit = self.params.get('ratelimit', None)
 280                 if rate_limit is None or byte_counter == 0:
 281                         return
 282                 now = time.time()
 283                 elapsed = now - start_time
 284                 if elapsed <= 0.0:
 285                         return
 286                 speed = float(byte_counter) / elapsed
 287                 if speed > rate_limit:
 288                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 289
 290         def report_destination(self, filename):
 291                 """Report destination filename."""
 292                 self.to_stdout(u'[download] Destination: %s' % filename)
 293
 294         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 295                 """Report download progress."""
 296                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 297                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 298
 299         def report_resuming_byte(self, resume_len):
 300                 """Report attemtp to resume at given byte."""
 301                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 302
 303         def report_file_already_downloaded(self, file_name):
 304                 """Report file has already been fully downloaded."""
 305                 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 306
 307         def report_unable_to_resume(self):
 308                 """Report it was impossible to resume download."""
 309                 self.to_stdout(u'[download] Unable to resume')
 310
 311         def report_finish(self):
 312                 """Report download finished."""
 313                 self.to_stdout(u'')
 314
 315         def process_info(self, info_dict):
 316                 """Process a single dictionary returned by an InfoExtractor."""
 317                 # Do nothing else if in simulate mode
 318                 if self.params.get('simulate', False):
 319                         # Verify URL if it's an HTTP one
 320                         if info_dict['url'].startswith('http'):
 321                                 try:
 322                                         info_dict['url'] = self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8')
 323                                 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 324                                         raise UnavailableFormatError
 325
 326                         # Forced printings
 327                         if self.params.get('forcetitle', False):
 328                                 print info_dict['title'].encode(preferredencoding())
 329                         if self.params.get('forceurl', False):
 330                                 print info_dict['url'].encode(preferredencoding())
 331
 332                         return
 333
 334                 try:
 335                         template_dict = dict(info_dict)
 336                         template_dict['epoch'] = unicode(long(time.time()))
 337                         filename = self.params['outtmpl'] % template_dict
 338                 except (ValueError, KeyError), err:
 339                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 340                 if self.params.get('nooverwrites', False) and os.path.exists(filename):
 341                         self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
 342                         return
 343
 344                 try:
 345                         self.pmkdir(filename)
 346                 except (OSError, IOError), err:
 347                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 348                         return
 349
 350                 try:
 351                         success = self._do_download(filename, info_dict['url'].encode('utf-8'))
 352                 except (OSError, IOError), err:
 353                         raise UnavailableFormatError
 354                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 355                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 356                         return
 357                 except (ContentTooShortError, ), err:
 358                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 359                         return
 360
 361                 if success:
 362                         try:
 363                                 self.post_process(filename, info_dict)
 364                         except (PostProcessingError), err:
 365                                 self.trouble('ERROR: postprocessing: %s' % str(err))
 366                                 return
 367
 368         def download(self, url_list):
 369                 """Download a given list of URLs."""
 370                 if len(url_list) > 1 and self.fixed_template():
 371                         raise SameFileError(self.params['outtmpl'])
 372
 373                 for url in url_list:
 374                         suitable_found = False
 375                         for ie in self._ies:
 376                                 # Go to next InfoExtractor if not suitable
 377                                 if not ie.suitable(url):
 378                                         continue
 379
 380                                 # Suitable InfoExtractor found
 381                                 suitable_found = True
 382
 383                                 # Extract information from URL and process it
 384                                 ie.extract(url)
 385
 386                                 # Suitable InfoExtractor had been found; go to next URL
 387                                 break
 388
 389                         if not suitable_found:
 390                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 391
 392                 return self._download_retcode
 393
 394         def post_process(self, filename, ie_info):
 395                 """Run the postprocessing chain on the given file."""
 396                 info = dict(ie_info)
 397                 info['filepath'] = filename
 398                 for pp in self._pps:
 399                         info = pp.run(info)
 400                         if info is None:
 401                                 break
 402
 403         def _download_with_rtmpdump(self, filename, url):
 404                 self.report_destination(filename)
 405
 406                 # Check for rtmpdump first
 407                 try:
 408                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 409                 except (OSError, IOError):
 410                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 411                         return False
 412
 413                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 414                 # the connection was interrumpted and resuming appears to be
 415                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 416                 retval = subprocess.call(['rtmpdump', '-q', '-r', url, '-o', filename] + [[], ['-e']][self.params.get('continuedl', False)])
 417                 while retval == 2:
 418                         self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename), skip_eol=True)
 419                         time.sleep(2.0) # This seems to be needed
 420                         retval = subprocess.call(['rtmpdump', '-q', '-e', '-r', url, '-o', filename])
 421                 if retval == 0:
 422                         self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
 423                         return True
 424                 else:
 425                         self.trouble('ERROR: rtmpdump exited with code %d' % retval)
 426                         return False
 427
 428         def _do_download(self, filename, url):
 429                 # Attempt to download using rtmpdump
 430                 if url.startswith('rtmp'):
 431                         return self._download_with_rtmpdump(filename, url)
 432
 433                 stream = None
 434                 open_mode = 'wb'
 435                 basic_request = urllib2.Request(url, None, std_headers)
 436                 request = urllib2.Request(url, None, std_headers)
 437
 438                 # Establish possible resume length
 439                 if os.path.isfile(filename):
 440                         resume_len = os.path.getsize(filename)
 441                 else:
 442                         resume_len = 0
 443
 444                 # Request parameters in case of being able to resume
 445                 if self.params.get('continuedl', False) and resume_len != 0:
 446                         self.report_resuming_byte(resume_len)
 447                         request.add_header('Range','bytes=%d-' % resume_len)
 448                         open_mode = 'ab'
 449
 450                 # Establish connection
 451                 try:
 452                         data = urllib2.urlopen(request)
 453                 except (urllib2.HTTPError, ), err:
 454                         if err.code != 416: #  416 is 'Requested range not satisfiable'
 455                                 raise
 456                         # Unable to resume
 457                         data = urllib2.urlopen(basic_request)
 458                         content_length = data.info()['Content-Length']
 459
 460                         if content_length is not None and long(content_length) == resume_len:
 461                                 # Because the file had already been fully downloaded
 462                                 self.report_file_already_downloaded(filename)
 463                                 return True
 464                         else:
 465                                 # Because the server didn't let us
 466                                 self.report_unable_to_resume()
 467                                 open_mode = 'wb'
 468
 469                 data_len = data.info().get('Content-length', None)
 470                 data_len_str = self.format_bytes(data_len)
 471                 byte_counter = 0
 472                 block_size = 1024
 473                 start = time.time()
 474                 while True:
 475                         # Download and write
 476                         before = time.time()
 477                         data_block = data.read(block_size)
 478                         after = time.time()
 479                         data_block_len = len(data_block)
 480                         if data_block_len == 0:
 481                                 break
 482                         byte_counter += data_block_len
 483
 484                         # Open file just in time
 485                         if stream is None:
 486                                 try:
 487                                         stream = open(filename, open_mode)
 488                                         self.report_destination(filename)
 489                                 except (OSError, IOError), err:
 490                                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 491                                         return False
 492                         stream.write(data_block)
 493                         block_size = self.best_block_size(after - before, data_block_len)
 494
 495                         # Progress message
 496                         percent_str = self.calc_percent(byte_counter, data_len)
 497                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 498                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 499                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 500
 501                         # Apply rate limit
 502                         self.slow_down(start, byte_counter)
 503
 504                 self.report_finish()
 505                 if data_len is not None and str(byte_counter) != data_len:
 506                         raise ContentTooShortError(byte_counter, long(data_len))
 507                 return True
 508
 509 class InfoExtractor(object):
 510         """Information Extractor class.
 511
 512         Information extractors are the classes that, given a URL, extract
 513         information from the video (or videos) the URL refers to. This
 514         information includes the real video URL, the video title and simplified
 515         title, author and others. The information is stored in a dictionary
 516         which is then passed to the FileDownloader. The FileDownloader
 517         processes this information possibly downloading the video to the file
 518         system, among other possible outcomes. The dictionaries must include
 519         the following fields:
 520
 521         id:             Video identifier.
 522         url:            Final video URL.
 523         uploader:       Nickname of the video uploader.
 524         title:          Literal title.
 525         stitle:         Simplified title.
 526         ext:            Video filename extension.
 527
 528         Subclasses of this one should re-define the _real_initialize() and
 529         _real_extract() methods, as well as the suitable() static method.
 530         Probably, they should also be instantiated and added to the main
 531         downloader.
 532         """
 533
 534         _ready = False
 535         _downloader = None
 536
 537         def __init__(self, downloader=None):
 538                 """Constructor. Receives an optional downloader."""
 539                 self._ready = False
 540                 self.set_downloader(downloader)
 541
 542         @staticmethod
 543         def suitable(url):
 544                 """Receives a URL and returns True if suitable for this IE."""
 545                 return False
 546
 547         def initialize(self):
 548                 """Initializes an instance (authentication, etc)."""
 549                 if not self._ready:
 550                         self._real_initialize()
 551                         self._ready = True
 552
 553         def extract(self, url):
 554                 """Extracts URL information and returns it in list of dicts."""
 555                 self.initialize()
 556                 return self._real_extract(url)
 557
 558         def set_downloader(self, downloader):
 559                 """Sets the downloader for this IE."""
 560                 self._downloader = downloader
 561
 562         def _real_initialize(self):
 563                 """Real initialization process. Redefine in subclasses."""
 564                 pass
 565
 566         def _real_extract(self, url):
 567                 """Real extraction process. Redefine in subclasses."""
 568                 pass
 569
 570 class YoutubeIE(InfoExtractor):
 571         """Information extractor for youtube.com."""
 572
 573         _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 574         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 575         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 576         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 577         _NETRC_MACHINE = 'youtube'
 578         _available_formats = ['37', '22', '35', '18', '5', '17', '13', None] # listed in order of priority for -b flag
 579         _video_extensions = {
 580                 '13': '3gp',
 581                 '17': 'mp4',
 582                 '18': 'mp4',
 583                 '22': 'mp4',
 584                 '37': 'mp4',
 585         }
 586
 587         @staticmethod
 588         def suitable(url):
 589                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 590
 591         @staticmethod
 592         def htmlentity_transform(matchobj):
 593                 """Transforms an HTML entity to a Unicode character."""
 594                 entity = matchobj.group(1)
 595
 596                 # Known non-numeric HTML entity
 597                 if entity in htmlentitydefs.name2codepoint:
 598                         return unichr(htmlentitydefs.name2codepoint[entity])
 599
 600                 # Unicode character
 601                 mobj = re.match(ur'(?u)#(x?\d+)', entity)
 602                 if mobj is not None:
 603                         numstr = mobj.group(1)
 604                         if numstr.startswith(u'x'):
 605                                 base = 16
 606                                 numstr = u'0%s' % numstr
 607                         else:
 608                                 base = 10
 609                         return unichr(long(numstr, base))
 610
 611                 # Unknown entity in name, return its literal representation
 612                 return (u'&%s;' % entity)
 613
 614         def report_lang(self):
 615                 """Report attempt to set language."""
 616                 self._downloader.to_stdout(u'[youtube] Setting language')
 617
 618         def report_login(self):
 619                 """Report attempt to log in."""
 620                 self._downloader.to_stdout(u'[youtube] Logging in')
 621
 622         def report_age_confirmation(self):
 623                 """Report attempt to confirm age."""
 624                 self._downloader.to_stdout(u'[youtube] Confirming age')
 625
 626         def report_video_info_webpage_download(self, video_id):
 627                 """Report attempt to download video info webpage."""
 628                 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
 629
 630         def report_information_extraction(self, video_id):
 631                 """Report attempt to extract video information."""
 632                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 633
 634         def report_unavailable_format(self, video_id, format):
 635                 """Report extracted video URL."""
 636                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 637
 638         def report_rtmp_download(self):
 639                 """Indicate the download will use the RTMP protocol."""
 640                 self._downloader.to_stdout(u'[youtube] RTMP download detected')
 641
 642         def _real_initialize(self):
 643                 if self._downloader is None:
 644                         return
 645
 646                 username = None
 647                 password = None
 648                 downloader_params = self._downloader.params
 649
 650                 # Attempt to use provided username and password or .netrc data
 651                 if downloader_params.get('username', None) is not None:
 652                         username = downloader_params['username']
 653                         password = downloader_params['password']
 654                 elif downloader_params.get('usenetrc', False):
 655                         try:
 656                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 657                                 if info is not None:
 658                                         username = info[0]
 659                                         password = info[2]
 660                                 else:
 661                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 662                         except (IOError, netrc.NetrcParseError), err:
 663                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 664                                 return
 665
 666                 # Set language
 667                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 668                 try:
 669                         self.report_lang()
 670                         urllib2.urlopen(request).read()
 671                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 672                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 673                         return
 674
 675                 # No authentication to be performed
 676                 if username is None:
 677                         return
 678
 679                 # Log in
 680                 login_form = {
 681                                 'current_form': 'loginForm',
 682                                 'next':         '/',
 683                                 'action_login': 'Log In',
 684                                 'username':     username,
 685                                 'password':     password,
 686                                 }
 687                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 688                 try:
 689                         self.report_login()
 690                         login_results = urllib2.urlopen(request).read()
 691                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 692                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 693                                 return
 694                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 695                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 696                         return
 697
 698                 # Confirm age
 699                 age_form = {
 700                                 'next_url':             '/',
 701                                 'action_confirm':       'Confirm',
 702                                 }
 703                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 704                 try:
 705                         self.report_age_confirmation()
 706                         age_results = urllib2.urlopen(request).read()
 707                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 708                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 709                         return
 710
 711         def _real_extract(self, url):
 712                 # Extract video id from URL
 713                 mobj = re.match(self._VALID_URL, url)
 714                 if mobj is None:
 715                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 716                         return
 717                 video_id = mobj.group(2)
 718
 719                 # Downloader parameters
 720                 best_quality = False
 721                 format_param = None
 722                 quality_index = 0
 723                 if self._downloader is not None:
 724                         params = self._downloader.params
 725                         format_param = params.get('format', None)
 726                         if format_param == '0':
 727                                 format_param = self._available_formats[quality_index]
 728                                 best_quality = True
 729
 730                 while True:
 731                         # Extension
 732                         video_extension = self._video_extensions.get(format_param, 'flv')
 733
 734                         # Get video info
 735                         video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id
 736                         request = urllib2.Request(video_info_url, None, std_headers)
 737                         try:
 738                                 self.report_video_info_webpage_download(video_id)
 739                                 video_info_webpage = urllib2.urlopen(request).read()
 740                                 video_info = parse_qs(video_info_webpage)
 741                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 742                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 743                                 return
 744                         self.report_information_extraction(video_id)
 745
 746                         # "t" param
 747                         if 'token' not in video_info:
 748                                 # Attempt to see if YouTube has issued an error message
 749                                 if 'reason' not in video_info:
 750                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
 751                                         stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
 752                                         stream.write(video_info_webpage)
 753                                         stream.close()
 754                                 else:
 755                                         reason = urllib.unquote_plus(video_info['reason'][0])
 756                                         self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
 757                                 return
 758                         token = urllib.unquote_plus(video_info['token'][0])
 759                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
 760                         if format_param is not None:
 761                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 762
 763                         # Check possible RTMP download
 764                         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 765                                 self.report_rtmp_download()
 766                                 video_real_url = video_info['conn'][0]
 767
 768                         # uploader
 769                         if 'author' not in video_info:
 770                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 771                                 return
 772                         video_uploader = urllib.unquote_plus(video_info['author'][0])
 773
 774                         # title
 775                         if 'title' not in video_info:
 776                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 777                                 return
 778                         video_title = urllib.unquote_plus(video_info['title'][0])
 779                         video_title = video_title.decode('utf-8')
 780                         video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
 781                         video_title = video_title.replace(os.sep, u'%')
 782
 783                         # simplified title
 784                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 785                         simple_title = simple_title.strip(ur'_')
 786
 787                         try:
 788                                 # Process video information
 789                                 self._downloader.process_info({
 790                                         'id':           video_id.decode('utf-8'),
 791                                         'url':          video_real_url.decode('utf-8'),
 792                                         'uploader':     video_uploader.decode('utf-8'),
 793                                         'title':        video_title,
 794                                         'stitle':       simple_title,
 795                                         'ext':          video_extension.decode('utf-8'),
 796                                 })
 797
 798                                 return
 799
 800                         except UnavailableFormatError, err:
 801                                 if best_quality:
 802                                         if quality_index == len(self._available_formats) - 1:
 803                                                 # I don't ever expect this to happen
 804                                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 805                                                 return
 806                                         else:
 807                                                 self.report_unavailable_format(video_id, format_param)
 808                                                 quality_index += 1
 809                                                 format_param = self._available_formats[quality_index]
 810                                                 continue
 811                                 else:
 812                                         self._downloader.trouble('ERROR: format not available for video')
 813                                         return
 814
 815
 816 class MetacafeIE(InfoExtractor):
 817         """Information Extractor for metacafe.com."""
 818
 819         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 820         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 821         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 822         _youtube_ie = None
 823
 824         def __init__(self, youtube_ie, downloader=None):
 825                 InfoExtractor.__init__(self, downloader)
 826                 self._youtube_ie = youtube_ie
 827
 828         @staticmethod
 829         def suitable(url):
 830                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 831
 832         def report_disclaimer(self):
 833                 """Report disclaimer retrieval."""
 834                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 835
 836         def report_age_confirmation(self):
 837                 """Report attempt to confirm age."""
 838                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 839
 840         def report_download_webpage(self, video_id):
 841                 """Report webpage download."""
 842                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
 843
 844         def report_extraction(self, video_id):
 845                 """Report information extraction."""
 846                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
 847
 848         def _real_initialize(self):
 849                 # Retrieve disclaimer
 850                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
 851                 try:
 852                         self.report_disclaimer()
 853                         disclaimer = urllib2.urlopen(request).read()
 854                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 855                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 856                         return
 857
 858                 # Confirm age
 859                 disclaimer_form = {
 860                         'filters': '0',
 861                         'submit': "Continue - I'm over 18",
 862                         }
 863                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
 864                 try:
 865                         self.report_age_confirmation()
 866                         disclaimer = urllib2.urlopen(request).read()
 867                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 868                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 869                         return
 870
 871         def _real_extract(self, url):
 872                 # Extract id and simplified title from URL
 873                 mobj = re.match(self._VALID_URL, url)
 874                 if mobj is None:
 875                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 876                         return
 877
 878                 video_id = mobj.group(1)
 879
 880                 # Check if video comes from YouTube
 881                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 882                 if mobj2 is not None:
 883                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 884                         return
 885
 886                 simple_title = mobj.group(2).decode('utf-8')
 887                 video_extension = 'flv'
 888
 889                 # Retrieve video webpage to extract further information
 890                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 891                 try:
 892                         self.report_download_webpage(video_id)
 893                         webpage = urllib2.urlopen(request).read()
 894                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 895                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 896                         return
 897
 898                 # Extract URL, uploader and title from webpage
 899                 self.report_extraction(video_id)
 900                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 901                 if mobj is None:
 902                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 903                         return
 904                 mediaURL = urllib.unquote(mobj.group(1))
 905
 906                 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 907                 #if mobj is None:
 908                 #       self._downloader.trouble(u'ERROR: unable to extract gdaKey')
 909                 #       return
 910                 #gdaKey = mobj.group(1)
 911                 #
 912                 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 913
 914                 video_url = mediaURL
 915
 916                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 917                 if mobj is None:
 918                         self._downloader.trouble(u'ERROR: unable to extract title')
 919                         return
 920                 video_title = mobj.group(1).decode('utf-8')
 921
 922                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
 923                 if mobj is None:
 924                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 925                         return
 926                 video_uploader = mobj.group(1)
 927
 928                 try:
 929                         # Process video information
 930                         self._downloader.process_info({
 931                                 'id':           video_id.decode('utf-8'),
 932                                 'url':          video_url.decode('utf-8'),
 933                                 'uploader':     video_uploader.decode('utf-8'),
 934                                 'title':        video_title,
 935                                 'stitle':       simple_title,
 936                                 'ext':          video_extension.decode('utf-8'),
 937                         })
 938                 except UnavailableFormatError:
 939                         self._downloader.trouble(u'ERROR: format not available for video')
 940
 941
 942 class GoogleIE(InfoExtractor):
 943         """Information extractor for video.google.com."""
 944
 945         _VALID_URL = r'(?:http://)?video\.google\.com/videoplay\?docid=([^\&]+).*'
 946
 947         def __init__(self, downloader=None):
 948                 InfoExtractor.__init__(self, downloader)
 949
 950         @staticmethod
 951         def suitable(url):
 952                 return (re.match(GoogleIE._VALID_URL, url) is not None)
 953
 954         def report_download_webpage(self, video_id):
 955                 """Report webpage download."""
 956                 self._downloader.to_stdout(u'[video.google] %s: Downloading webpage' % video_id)
 957
 958         def report_extraction(self, video_id):
 959                 """Report information extraction."""
 960                 self._downloader.to_stdout(u'[video.google] %s: Extracting information' % video_id)
 961
 962         def _real_initialize(self):
 963                 return
 964
 965         def _real_extract(self, url):
 966                 # Extract id from URL
 967                 mobj = re.match(self._VALID_URL, url)
 968                 if mobj is None:
 969                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 970                         return
 971
 972                 video_id = mobj.group(1)
 973
 974                 video_extension = 'mp4'
 975
 976                 # Retrieve video webpage to extract further information
 977                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s' % video_id)
 978                 try:
 979                         self.report_download_webpage(video_id)
 980                         webpage = urllib2.urlopen(request).read()
 981                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 982                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 983                         return
 984
 985                 # Extract URL, uploader, and title from webpage
 986                 self.report_extraction(video_id)
 987                 mobj = re.search(r"download_url:'(.*)'", webpage)
 988                 if mobj is None:
 989                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 990                         return
 991                 mediaURL = urllib.unquote(mobj.group(1))
 992                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
 993                 mediaURL = mediaURL.replace('\\x26', '\x26')
 994
 995                 video_url = mediaURL
 996
 997                 mobj = re.search(r'<title>(.*)</title>', webpage)
 998                 if mobj is None:
 999                         self._downloader.trouble(u'ERROR: unable to extract title')
1000                         return
1001                 video_title = mobj.group(1).decode('utf-8')
1002
1003                 # Google Video doesn't show uploader nicknames?
1004                 video_uploader = 'uploader'
1005
1006                 try:
1007                         # Process video information
1008                         self._downloader.process_info({
1009                                 'id':           video_id.decode('utf-8'),
1010                                 'url':          video_url.decode('utf-8'),
1011                                 'uploader':     video_uploader.decode('utf-8'),
1012                                 'title':        video_title.decode('utf-8'),
1013                                 'stitle':       video_title.decode('utf-8'),
1014                                 'ext':          video_extension.decode('utf-8'),
1015                         })
1016                 except UnavailableFormatError:
1017                         self._downloader.trouble(u'ERROR: format not available for video')
1018
1019
1020 class PhotobucketIE(InfoExtractor):
1021         """Information extractor for photobucket.com."""
1022
1023         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1024
1025         def __init__(self, downloader=None):
1026                 InfoExtractor.__init__(self, downloader)
1027
1028         @staticmethod
1029         def suitable(url):
1030                 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1031
1032         def report_download_webpage(self, video_id):
1033                 """Report webpage download."""
1034                 self._downloader.to_stdout(u'[photobucket] %s: Downloading webpage' % video_id)
1035
1036         def report_extraction(self, video_id):
1037                 """Report information extraction."""
1038                 self._downloader.to_stdout(u'[photobucket] %s: Extracting information' % video_id)
1039
1040         def _real_initialize(self):
1041                 return
1042
1043         def _real_extract(self, url):
1044                 # Extract id from URL
1045                 mobj = re.match(self._VALID_URL, url)
1046                 if mobj is None:
1047                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1048                         return
1049
1050                 video_id = mobj.group(1)
1051
1052                 video_extension = 'flv'
1053
1054                 # Retrieve video webpage to extract further information
1055                 request = urllib2.Request(url)
1056                 try:
1057                         self.report_download_webpage(video_id)
1058                         webpage = urllib2.urlopen(request).read()
1059                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1060                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1061                         return
1062
1063                 # Extract URL, uploader, and title from webpage
1064                 self.report_extraction(video_id)
1065                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1066                 if mobj is None:
1067                         self._downloader.trouble(u'ERROR: unable to extract media URL')
1068                         return
1069                 mediaURL = urllib.unquote(mobj.group(1))
1070
1071                 video_url = mediaURL
1072
1073                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1074                 if mobj is None:
1075                         self._downloader.trouble(u'ERROR: unable to extract title')
1076                         return
1077                 video_title = mobj.group(1).decode('utf-8')
1078
1079                 video_uploader = mobj.group(2).decode('utf-8')
1080
1081                 try:
1082                         # Process video information
1083                         self._downloader.process_info({
1084                                 'id':           video_id.decode('utf-8'),
1085                                 'url':          video_url.decode('utf-8'),
1086                                 'uploader':     video_uploader.decode('utf-8'),
1087                                 'title':        video_title.decode('utf-8'),
1088                                 'stitle':       video_title.decode('utf-8'),
1089                                 'ext':          video_extension.decode('utf-8'),
1090                         })
1091                 except UnavailableFormatError:
1092                         self._downloader.trouble(u'ERROR: format not available for video')
1093
1094
1095 class YoutubeSearchIE(InfoExtractor):
1096         """Information Extractor for YouTube search queries."""
1097         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1098         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1099         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1100         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1101         _youtube_ie = None
1102         _max_youtube_results = 1000
1103
1104         def __init__(self, youtube_ie, downloader=None):
1105                 InfoExtractor.__init__(self, downloader)
1106                 self._youtube_ie = youtube_ie
1107
1108         @staticmethod
1109         def suitable(url):
1110                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1111
1112         def report_download_page(self, query, pagenum):
1113                 """Report attempt to download playlist page with given number."""
1114                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1115
1116         def _real_initialize(self):
1117                 self._youtube_ie.initialize()
1118
1119         def _real_extract(self, query):
1120                 mobj = re.match(self._VALID_QUERY, query)
1121                 if mobj is None:
1122                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1123                         return
1124
1125                 prefix, query = query.split(':')
1126                 prefix = prefix[8:]
1127                 if prefix == '':
1128                         self._download_n_results(query, 1)
1129                         return
1130                 elif prefix == 'all':
1131                         self._download_n_results(query, self._max_youtube_results)
1132                         return
1133                 else:
1134                         try:
1135                                 n = long(prefix)
1136                                 if n <= 0:
1137                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1138                                         return
1139                                 elif n > self._max_youtube_results:
1140                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
1141                                         n = self._max_youtube_results
1142                                 self._download_n_results(query, n)
1143                                 return
1144                         except ValueError: # parsing prefix as integer fails
1145                                 self._download_n_results(query, 1)
1146                                 return
1147
1148         def _download_n_results(self, query, n):
1149                 """Downloads a specified number of results for a query"""
1150
1151                 video_ids = []
1152                 already_seen = set()
1153                 pagenum = 1
1154
1155                 while True:
1156                         self.report_download_page(query, pagenum)
1157                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1158                         request = urllib2.Request(result_url, None, std_headers)
1159                         try:
1160                                 page = urllib2.urlopen(request).read()
1161                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1162                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1163                                 return
1164
1165                         # Extract video identifiers
1166                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1167                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1168                                 if video_id not in already_seen:
1169                                         video_ids.append(video_id)
1170                                         already_seen.add(video_id)
1171                                         if len(video_ids) == n:
1172                                                 # Specified n videos reached
1173                                                 for id in video_ids:
1174                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1175                                                 return
1176
1177                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1178                                 for id in video_ids:
1179                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1180                                 return
1181
1182                         pagenum = pagenum + 1
1183
1184 class YoutubePlaylistIE(InfoExtractor):
1185         """Information Extractor for YouTube playlists."""
1186
1187         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:view_play_list|my_playlists)\?.*?p=([^&]+).*'
1188         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1189         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1190         _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
1191         _youtube_ie = None
1192
1193         def __init__(self, youtube_ie, downloader=None):
1194                 InfoExtractor.__init__(self, downloader)
1195                 self._youtube_ie = youtube_ie
1196
1197         @staticmethod
1198         def suitable(url):
1199                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1200
1201         def report_download_page(self, playlist_id, pagenum):
1202                 """Report attempt to download playlist page with given number."""
1203                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1204
1205         def _real_initialize(self):
1206                 self._youtube_ie.initialize()
1207
1208         def _real_extract(self, url):
1209                 # Extract playlist id
1210                 mobj = re.match(self._VALID_URL, url)
1211                 if mobj is None:
1212                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1213                         return
1214
1215                 # Download playlist pages
1216                 playlist_id = mobj.group(1)
1217                 video_ids = []
1218                 pagenum = 1
1219
1220                 while True:
1221                         self.report_download_page(playlist_id, pagenum)
1222                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1223                         try:
1224                                 page = urllib2.urlopen(request).read()
1225                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1226                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1227                                 return
1228
1229                         # Extract video identifiers
1230                         ids_in_page = []
1231                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1232                                 if mobj.group(1) not in ids_in_page:
1233                                         ids_in_page.append(mobj.group(1))
1234                         video_ids.extend(ids_in_page)
1235
1236                         if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page:
1237                                 break
1238                         pagenum = pagenum + 1
1239
1240                 for id in video_ids:
1241                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1242                 return
1243
1244 class YoutubeUserIE(InfoExtractor):
1245         """Information Extractor for YouTube users."""
1246
1247         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
1248         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1249         _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
1250         _youtube_ie = None
1251
1252         def __init__(self, youtube_ie, downloader=None):
1253                 InfoExtractor.__init__(self, downloader)
1254                 self._youtube_ie = youtube_ie
1255
1256         @staticmethod
1257         def suitable(url):
1258                 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
1259
1260         def report_download_page(self, username):
1261                 """Report attempt to download user page."""
1262                 self._downloader.to_stdout(u'[youtube] user %s: Downloading page ' % (username))
1263
1264         def _real_initialize(self):
1265                 self._youtube_ie.initialize()
1266
1267         def _real_extract(self, url):
1268                 # Extract username
1269                 mobj = re.match(self._VALID_URL, url)
1270                 if mobj is None:
1271                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1272                         return
1273
1274                 # Download user page
1275                 username = mobj.group(1)
1276                 video_ids = []
1277                 pagenum = 1
1278
1279                 self.report_download_page(username)
1280                 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
1281                 try:
1282                         page = urllib2.urlopen(request).read()
1283                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1284                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1285                         return
1286
1287                 # Extract video identifiers
1288                 ids_in_page = []
1289
1290                 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1291                         if mobj.group(1) not in ids_in_page:
1292                                 ids_in_page.append(mobj.group(1))
1293                 video_ids.extend(ids_in_page)
1294
1295                 for id in video_ids:
1296                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1297                 return
1298
1299 class PostProcessor(object):
1300         """Post Processor class.
1301
1302         PostProcessor objects can be added to downloaders with their
1303         add_post_processor() method. When the downloader has finished a
1304         successful download, it will take its internal chain of PostProcessors
1305         and start calling the run() method on each one of them, first with
1306         an initial argument and then with the returned value of the previous
1307         PostProcessor.
1308
1309         The chain will be stopped if one of them ever returns None or the end
1310         of the chain is reached.
1311
1312         PostProcessor objects follow a "mutual registration" process similar
1313         to InfoExtractor objects.
1314         """
1315
1316         _downloader = None
1317
1318         def __init__(self, downloader=None):
1319                 self._downloader = downloader
1320
1321         def set_downloader(self, downloader):
1322                 """Sets the downloader for this PP."""
1323                 self._downloader = downloader
1324
1325         def run(self, information):
1326                 """Run the PostProcessor.
1327
1328                 The "information" argument is a dictionary like the ones
1329                 composed by InfoExtractors. The only difference is that this
1330                 one has an extra field called "filepath" that points to the
1331                 downloaded file.
1332
1333                 When this method returns None, the postprocessing chain is
1334                 stopped. However, this method may return an information
1335                 dictionary that will be passed to the next postprocessing
1336                 object in the chain. It can be the one it received after
1337                 changing some fields.
1338
1339                 In addition, this method may raise a PostProcessingError
1340                 exception that will be taken into account by the downloader
1341                 it was called from.
1342                 """
1343                 return information # by default, do nothing
1344
1345 ### MAIN PROGRAM ###
1346 if __name__ == '__main__':
1347         try:
1348                 # Modules needed only when running the main program
1349                 import getpass
1350                 import optparse
1351
1352                 # Function to update the program file with the latest version from bitbucket.org
1353                 def update_self(downloader, filename):
1354                         # Note: downloader only used for options
1355                         if not os.access (filename, os.W_OK):
1356                                 sys.exit('ERROR: no write permissions on %s' % filename)
1357
1358                         downloader.to_stdout('Updating to latest stable version...')
1359                         latest_url = 'http://bitbucket.org/rg3/youtube-dl/raw/tip/LATEST_VERSION'
1360                         latest_version = urllib.urlopen(latest_url).read().strip()
1361                         prog_url = 'http://bitbucket.org/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
1362                         newcontent = urllib.urlopen(prog_url).read()
1363                         stream = open(filename, 'w')
1364                         stream.write(newcontent)
1365                         stream.close()
1366                         downloader.to_stdout('Updated to version %s' % latest_version)
1367
1368                 # General configuration
1369                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1370                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1371                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1372
1373                 # Parse command line
1374                 parser = optparse.OptionParser(
1375                         usage='Usage: %prog [options] url...',
1376                         version='2010.01.06',
1377                         conflict_handler='resolve',
1378                 )
1379
1380                 parser.add_option('-h', '--help',
1381                                 action='help', help='print this help text and exit')
1382                 parser.add_option('-v', '--version',
1383                                 action='version', help='print program version and exit')
1384                 parser.add_option('-U', '--update',
1385                                 action='store_true', dest='update_self', help='update this program to latest stable version')
1386                 parser.add_option('-i', '--ignore-errors',
1387                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1388                 parser.add_option('-r', '--rate-limit',
1389                                 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1390
1391                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1392                 authentication.add_option('-u', '--username',
1393                                 dest='username', metavar='UN', help='account username')
1394                 authentication.add_option('-p', '--password',
1395                                 dest='password', metavar='PW', help='account password')
1396                 authentication.add_option('-n', '--netrc',
1397                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1398                 parser.add_option_group(authentication)
1399
1400                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1401                 video_format.add_option('-f', '--format',
1402                                 action='store', dest='format', metavar='FMT', help='video format code')
1403                 video_format.add_option('-b', '--best-quality',
1404                                 action='store_const', dest='format', help='download the best quality video possible', const='0')
1405                 video_format.add_option('-m', '--mobile-version',
1406                                 action='store_const', dest='format', help='alias for -f 17', const='17')
1407                 video_format.add_option('-d', '--high-def',
1408                                 action='store_const', dest='format', help='alias for -f 22', const='22')
1409                 parser.add_option_group(video_format)
1410
1411                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1412                 verbosity.add_option('-q', '--quiet',
1413                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1414                 verbosity.add_option('-s', '--simulate',
1415                                 action='store_true', dest='simulate', help='do not download video', default=False)
1416                 verbosity.add_option('-g', '--get-url',
1417                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1418                 verbosity.add_option('-e', '--get-title',
1419                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1420                 parser.add_option_group(verbosity)
1421
1422                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1423                 filesystem.add_option('-t', '--title',
1424                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
1425                 filesystem.add_option('-l', '--literal',
1426                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1427                 filesystem.add_option('-o', '--output',
1428                                 dest='outtmpl', metavar='TPL', help='output filename template')
1429                 filesystem.add_option('-a', '--batch-file',
1430                                 dest='batchfile', metavar='F', help='file containing URLs to download')
1431                 filesystem.add_option('-w', '--no-overwrites',
1432                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1433                 filesystem.add_option('-c', '--continue',
1434                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1435                 parser.add_option_group(filesystem)
1436
1437                 (opts, args) = parser.parse_args()
1438
1439                 # Batch file verification
1440                 batchurls = []
1441                 if opts.batchfile is not None:
1442                         try:
1443                                 batchurls = open(opts.batchfile, 'r').readlines()
1444                                 batchurls = [x.strip() for x in batchurls]
1445                                 batchurls = [x for x in batchurls if len(x) > 0]
1446                         except IOError:
1447                                 sys.exit(u'ERROR: batch file could not be read')
1448                 all_urls = batchurls + args
1449
1450                 # Conflicting, missing and erroneous options
1451                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1452                         parser.error(u'using .netrc conflicts with giving username/password')
1453                 if opts.password is not None and opts.username is None:
1454                         parser.error(u'account username missing')
1455                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1456                         parser.error(u'using output template conflicts with using title or literal title')
1457                 if opts.usetitle and opts.useliteral:
1458                         parser.error(u'using title conflicts with using literal title')
1459                 if opts.username is not None and opts.password is None:
1460                         opts.password = getpass.getpass(u'Type account password and press return:')
1461                 if opts.ratelimit is not None:
1462                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1463                         if numeric_limit is None:
1464                                 parser.error(u'invalid rate limit specified')
1465                         opts.ratelimit = numeric_limit
1466
1467                 # Information extractors
1468                 youtube_ie = YoutubeIE()
1469                 metacafe_ie = MetacafeIE(youtube_ie)
1470                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1471                 youtube_user_ie = YoutubeUserIE(youtube_ie)
1472                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1473                 google_ie = GoogleIE()
1474                 photobucket_ie = PhotobucketIE()
1475
1476                 # File downloader
1477                 fd = FileDownloader({
1478                         'usenetrc': opts.usenetrc,
1479                         'username': opts.username,
1480                         'password': opts.password,
1481                         'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1482                         'forceurl': opts.geturl,
1483                         'forcetitle': opts.gettitle,
1484                         'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1485                         'format': opts.format,
1486                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
1487                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1488                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1489                                 or u'%(id)s.%(ext)s'),
1490                         'ignoreerrors': opts.ignoreerrors,
1491                         'ratelimit': opts.ratelimit,
1492                         'nooverwrites': opts.nooverwrites,
1493                         'continuedl': opts.continue_dl,
1494                         })
1495                 fd.add_info_extractor(youtube_search_ie)
1496                 fd.add_info_extractor(youtube_pl_ie)
1497                 fd.add_info_extractor(youtube_user_ie)
1498                 fd.add_info_extractor(metacafe_ie)
1499                 fd.add_info_extractor(youtube_ie)
1500                 fd.add_info_extractor(google_ie)
1501                 fd.add_info_extractor(photobucket_ie)
1502
1503                 # Update version
1504                 if opts.update_self:
1505                         update_self(fd, sys.argv[0])
1506
1507                 # Maybe do nothing
1508                 if len(all_urls) < 1:
1509                         if not opts.update_self:
1510                                 parser.error(u'you must provide at least one URL')
1511                         else:
1512                                 sys.exit()
1513                 retcode = fd.download(all_urls)
1514                 sys.exit(retcode)
1515
1516         except DownloadError:
1517                 sys.exit(1)
1518         except SameFileError:
1519                 sys.exit(u'ERROR: fixed output name but more than one file to download')
1520         except KeyboardInterrupt:
1521                 sys.exit(u'\nERROR: Interrupted by user')