_ Git - youtube-dl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # License: Public domain code
   6 import htmlentitydefs
   7 import httplib
   8 import locale
   9 import math
  10 import netrc
  11 import os
  12 import os.path
  13 import re
  14 import socket
  15 import string
  16 import sys
  17 import time
  18 import urllib
  19 import urllib2
  20
  21 std_headers = {
  22         'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2',
  23         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  24         'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
  25         'Accept-Language': 'en-us,en;q=0.5',
  26 }
  27
  28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  29
  30 class DownloadError(Exception):
  31         """Download Error exception.
  32
  33         This exception may be thrown by FileDownloader objects if they are not
  34         configured to continue on errors. They will contain the appropriate
  35         error message.
  36         """
  37         pass
  38
  39 class SameFileError(Exception):
  40         """Same File exception.
  41
  42         This exception will be thrown by FileDownloader objects if they detect
  43         multiple files would have to be downloaded to the same file on disk.
  44         """
  45         pass
  46
  47 class PostProcessingError(Exception):
  48         """Post Processing exception.
  49
  50         This exception may be raised by PostProcessor's .run() method to
  51         indicate an error in the postprocessing task.
  52         """
  53         pass
  54
  55 class UnavailableFormatError(Exception):
  56         """Unavailable Format exception.
  57
  58         This exception will be thrown when a video is requested
  59         in a format that is not available for that video.
  60         """
  61         pass
  62
  63 class ContentTooShortError(Exception):
  64         """Content Too Short exception.
  65
  66         This exception may be raised by FileDownloader objects when a file they
  67         download is too small for what the server announced first, indicating
  68         the connection was probably interrupted.
  69         """
  70         # Both in bytes
  71         downloaded = None
  72         expected = None
  73
  74         def __init__(self, downloaded, expected):
  75                 self.downloaded = downloaded
  76                 self.expected = expected
  77
  78 class FileDownloader(object):
  79         """File Downloader class.
  80
  81         File downloader objects are the ones responsible of downloading the
  82         actual video file and writing it to disk if the user has requested
  83         it, among some other tasks. In most cases there should be one per
  84         program. As, given a video URL, the downloader doesn't know how to
  85         extract all the needed information, task that InfoExtractors do, it
  86         has to pass the URL to one of them.
  87
  88         For this, file downloader objects have a method that allows
  89         InfoExtractors to be registered in a given order. When it is passed
  90         a URL, the file downloader handles it to the first InfoExtractor it
  91         finds that reports being able to handle it. The InfoExtractor extracts
  92         all the information about the video or videos the URL refers to, and
  93         asks the FileDownloader to process the video information, possibly
  94         downloading the video.
  95
  96         File downloaders accept a lot of parameters. In order not to saturate
  97         the object constructor with arguments, it receives a dictionary of
  98         options instead. These options are available through the params
  99         attribute for the InfoExtractors to use. The FileDownloader also
 100         registers itself as the downloader in charge for the InfoExtractors
 101         that are added to it, so this is a "mutual registration".
 102
 103         Available options:
 104
 105         username:       Username for authentication purposes.
 106         password:       Password for authentication purposes.
 107         usenetrc:       Use netrc for authentication instead.
 108         quiet:          Do not print messages to stdout.
 109         forceurl:       Force printing final URL.
 110         forcetitle:     Force printing title.
 111         simulate:       Do not download the video files.
 112         format:         Video format code.
 113         outtmpl:        Template for output names.
 114         ignoreerrors:   Do not stop on download errors.
 115         ratelimit:      Download speed limit, in bytes/sec.
 116         nooverwrites:   Prevent overwriting files.
 117         continuedl:     Try to continue downloads if possible.
 118         """
 119
 120         params = None
 121         _ies = []
 122         _pps = []
 123         _download_retcode = None
 124
 125         def __init__(self, params):
 126                 """Create a FileDownloader object with the given options."""
 127                 self._ies = []
 128                 self._pps = []
 129                 self._download_retcode = 0
 130                 self.params = params
 131
 132         @staticmethod
 133         def pmkdir(filename):
 134                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 135                 components = filename.split(os.sep)
 136                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 137                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 138                 for dir in aggregate:
 139                         if not os.path.exists(dir):
 140                                 os.mkdir(dir)
 141
 142         @staticmethod
 143         def format_bytes(bytes):
 144                 if bytes is None:
 145                         return 'N/A'
 146                 if type(bytes) is str:
 147                         bytes = float(bytes)
 148                 if bytes == 0.0:
 149                         exponent = 0
 150                 else:
 151                         exponent = long(math.log(bytes, 1024.0))
 152                 suffix = 'bkMGTPEZY'[exponent]
 153                 converted = float(bytes) / float(1024**exponent)
 154                 return '%.2f%s' % (converted, suffix)
 155
 156         @staticmethod
 157         def calc_percent(byte_counter, data_len):
 158                 if data_len is None:
 159                         return '---.-%'
 160                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 161
 162         @staticmethod
 163         def calc_eta(start, now, total, current):
 164                 if total is None:
 165                         return '--:--'
 166                 dif = now - start
 167                 if current == 0 or dif < 0.001: # One millisecond
 168                         return '--:--'
 169                 rate = float(current) / dif
 170                 eta = long((float(total) - float(current)) / rate)
 171                 (eta_mins, eta_secs) = divmod(eta, 60)
 172                 if eta_mins > 99:
 173                         return '--:--'
 174                 return '%02d:%02d' % (eta_mins, eta_secs)
 175
 176         @staticmethod
 177         def calc_speed(start, now, bytes):
 178                 dif = now - start
 179                 if bytes == 0 or dif < 0.001: # One millisecond
 180                         return '%10s' % '---b/s'
 181                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 182
 183         @staticmethod
 184         def best_block_size(elapsed_time, bytes):
 185                 new_min = max(bytes / 2.0, 1.0)
 186                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 187                 if elapsed_time < 0.001:
 188                         return long(new_max)
 189                 rate = bytes / elapsed_time
 190                 if rate > new_max:
 191                         return long(new_max)
 192                 if rate < new_min:
 193                         return long(new_min)
 194                 return long(rate)
 195
 196         @staticmethod
 197         def parse_bytes(bytestr):
 198                 """Parse a string indicating a byte quantity into a long integer."""
 199                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 200                 if matchobj is None:
 201                         return None
 202                 number = float(matchobj.group(1))
 203                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 204                 return long(round(number * multiplier))
 205
 206         @staticmethod
 207         def verify_url(url):
 208                 """Verify a URL is valid and data could be downloaded."""
 209                 request = urllib2.Request(url, None, std_headers)
 210                 data = urllib2.urlopen(request)
 211                 data.read(1)
 212                 data.close()
 213
 214         def add_info_extractor(self, ie):
 215                 """Add an InfoExtractor object to the end of the list."""
 216                 self._ies.append(ie)
 217                 ie.set_downloader(self)
 218
 219         def add_post_processor(self, pp):
 220                 """Add a PostProcessor object to the end of the chain."""
 221                 self._pps.append(pp)
 222                 pp.set_downloader(self)
 223
 224         def to_stdout(self, message, skip_eol=False):
 225                 """Print message to stdout if not in quiet mode."""
 226                 if not self.params.get('quiet', False):
 227                         print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
 228                         sys.stdout.flush()
 229
 230         def to_stderr(self, message):
 231                 """Print message to stderr."""
 232                 print >>sys.stderr, message.encode(locale.getpreferredencoding())
 233
 234         def fixed_template(self):
 235                 """Checks if the output template is fixed."""
 236                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 237
 238         def trouble(self, message=None):
 239                 """Determine action to take when a download problem appears.
 240
 241                 Depending on if the downloader has been configured to ignore
 242                 download errors or not, this method may throw an exception or
 243                 not when errors are found, after printing the message.
 244                 """
 245                 if message is not None:
 246                         self.to_stderr(message)
 247                 if not self.params.get('ignoreerrors', False):
 248                         raise DownloadError(message)
 249                 self._download_retcode = 1
 250
 251         def slow_down(self, start_time, byte_counter):
 252                 """Sleep if the download speed is over the rate limit."""
 253                 rate_limit = self.params.get('ratelimit', None)
 254                 if rate_limit is None or byte_counter == 0:
 255                         return
 256                 now = time.time()
 257                 elapsed = now - start_time
 258                 if elapsed <= 0.0:
 259                         return
 260                 speed = float(byte_counter) / elapsed
 261                 if speed > rate_limit:
 262                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 263
 264         def report_destination(self, filename):
 265                 """Report destination filename."""
 266                 self.to_stdout(u'[download] Destination: %s' % filename)
 267
 268         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 269                 """Report download progress."""
 270                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 271                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 272
 273         def report_resuming_byte(self, resume_len):
 274                 """Report attemtp to resume at given byte."""
 275                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 276
 277         def report_file_already_downloaded(self, file_name):
 278                 """Report file has already been fully downloaded."""
 279                 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 280
 281         def report_unable_to_resume(self):
 282                 """Report it was impossible to resume download."""
 283                 self.to_stdout(u'[download] Unable to resume')
 284
 285         def report_finish(self):
 286                 """Report download finished."""
 287                 self.to_stdout(u'')
 288
 289         def process_info(self, info_dict):
 290                 """Process a single dictionary returned by an InfoExtractor."""
 291                 # Do nothing else if in simulate mode
 292                 if self.params.get('simulate', False):
 293                         try:
 294                                 self.verify_url(info_dict['url'])
 295                         except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 296                                 raise UnavailableFormatError
 297
 298                         # Forced printings
 299                         if self.params.get('forcetitle', False):
 300                                 print info_dict['title'].encode(locale.getpreferredencoding())
 301                         if self.params.get('forceurl', False):
 302                                 print info_dict['url'].encode(locale.getpreferredencoding())
 303
 304                         return
 305
 306                 try:
 307                         template_dict = dict(info_dict)
 308                         template_dict['epoch'] = unicode(long(time.time()))
 309                         filename = self.params['outtmpl'] % template_dict
 310                 except (ValueError, KeyError), err:
 311                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 312                 if self.params['nooverwrites'] and os.path.exists(filename):
 313                         self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
 314                         return
 315
 316                 try:
 317                         self.pmkdir(filename)
 318                 except (OSError, IOError), err:
 319                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 320                         return
 321
 322                 try:
 323                         success = self._do_download(filename, info_dict['url'])
 324                 except (OSError, IOError), err:
 325                         raise UnavailableFormatError
 326                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 327                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 328                         return
 329                 except (ContentTooShortError, ), err:
 330                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 331                         return
 332
 333                 if success:
 334                         try:
 335                                 self.post_process(filename, info_dict)
 336                         except (PostProcessingError), err:
 337                                 self.trouble('ERROR: postprocessing: %s' % str(err))
 338                                 return
 339
 340         def download(self, url_list):
 341                 """Download a given list of URLs."""
 342                 if len(url_list) > 1 and self.fixed_template():
 343                         raise SameFileError(self.params['outtmpl'])
 344
 345                 for url in url_list:
 346                         suitable_found = False
 347                         for ie in self._ies:
 348                                 # Go to next InfoExtractor if not suitable
 349                                 if not ie.suitable(url):
 350                                         continue
 351
 352                                 # Suitable InfoExtractor found
 353                                 suitable_found = True
 354
 355                                 # Extract information from URL and process it
 356                                 ie.extract(url)
 357
 358                                 # Suitable InfoExtractor had been found; go to next URL
 359                                 break
 360
 361                         if not suitable_found:
 362                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 363
 364                 return self._download_retcode
 365
 366         def post_process(self, filename, ie_info):
 367                 """Run the postprocessing chain on the given file."""
 368                 info = dict(ie_info)
 369                 info['filepath'] = filename
 370                 for pp in self._pps:
 371                         info = pp.run(info)
 372                         if info is None:
 373                                 break
 374
 375         def _do_download(self, filename, url):
 376                 stream = None
 377                 open_mode = 'ab'
 378
 379                 basic_request = urllib2.Request(url, None, std_headers)
 380                 request = urllib2.Request(url, None, std_headers)
 381
 382                 # Attempt to resume download with "continuedl" option
 383                 if os.path.isfile(filename):
 384                         resume_len = os.path.getsize(filename)
 385                 else:
 386                         resume_len = 0
 387                 if self.params['continuedl'] and resume_len != 0:
 388                         self.report_resuming_byte(resume_len)
 389                         request.add_header('Range','bytes=%d-' % resume_len)
 390
 391                 # Establish connection
 392                 try:
 393                         data = urllib2.urlopen(request)
 394                 except (urllib2.HTTPError, ), err:
 395                         if err.code != 416: #  416 is 'Requested range not satisfiable'
 396                                 raise
 397                         data = urllib2.urlopen(basic_request)
 398                         content_length = data.info()['Content-Length']
 399                         if content_length is not None and long(content_length) == resume_len:
 400                                 self.report_file_already_downloaded(filename)
 401                                 return True
 402                         else:
 403                                 self.report_unable_to_resume()
 404                                 open_mode = 'wb'
 405
 406                 data_len = data.info().get('Content-length', None)
 407                 data_len_str = self.format_bytes(data_len)
 408                 byte_counter = 0
 409                 block_size = 1024
 410                 start = time.time()
 411                 while True:
 412                         # Download and write
 413                         before = time.time()
 414                         data_block = data.read(block_size)
 415                         after = time.time()
 416                         data_block_len = len(data_block)
 417                         if data_block_len == 0:
 418                                 break
 419                         byte_counter += data_block_len
 420
 421                         # Open file just in time
 422                         if stream is None:
 423                                 try:
 424                                         stream = open(filename, open_mode)
 425                                         self.report_destination(filename)
 426                                 except (OSError, IOError), err:
 427                                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 428                                         return False
 429                         stream.write(data_block)
 430                         block_size = self.best_block_size(after - before, data_block_len)
 431
 432                         # Progress message
 433                         percent_str = self.calc_percent(byte_counter, data_len)
 434                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 435                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 436                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 437
 438                         # Apply rate limit
 439                         self.slow_down(start, byte_counter)
 440
 441                 self.report_finish()
 442                 if data_len is not None and str(byte_counter) != data_len:
 443                         raise ContentTooShortError(byte_counter, long(data_len))
 444                 return True
 445
 446 class InfoExtractor(object):
 447         """Information Extractor class.
 448
 449         Information extractors are the classes that, given a URL, extract
 450         information from the video (or videos) the URL refers to. This
 451         information includes the real video URL, the video title and simplified
 452         title, author and others. The information is stored in a dictionary
 453         which is then passed to the FileDownloader. The FileDownloader
 454         processes this information possibly downloading the video to the file
 455         system, among other possible outcomes. The dictionaries must include
 456         the following fields:
 457
 458         id:             Video identifier.
 459         url:            Final video URL.
 460         uploader:       Nickname of the video uploader.
 461         title:          Literal title.
 462         stitle:         Simplified title.
 463         ext:            Video filename extension.
 464
 465         Subclasses of this one should re-define the _real_initialize() and
 466         _real_extract() methods, as well as the suitable() static method.
 467         Probably, they should also be instantiated and added to the main
 468         downloader.
 469         """
 470
 471         _ready = False
 472         _downloader = None
 473
 474         def __init__(self, downloader=None):
 475                 """Constructor. Receives an optional downloader."""
 476                 self._ready = False
 477                 self.set_downloader(downloader)
 478
 479         @staticmethod
 480         def suitable(url):
 481                 """Receives a URL and returns True if suitable for this IE."""
 482                 return False
 483
 484         def initialize(self):
 485                 """Initializes an instance (authentication, etc)."""
 486                 if not self._ready:
 487                         self._real_initialize()
 488                         self._ready = True
 489
 490         def extract(self, url):
 491                 """Extracts URL information and returns it in list of dicts."""
 492                 self.initialize()
 493                 return self._real_extract(url)
 494
 495         def set_downloader(self, downloader):
 496                 """Sets the downloader for this IE."""
 497                 self._downloader = downloader
 498
 499         def _real_initialize(self):
 500                 """Real initialization process. Redefine in subclasses."""
 501                 pass
 502
 503         def _real_extract(self, url):
 504                 """Real extraction process. Redefine in subclasses."""
 505                 pass
 506
 507 class YoutubeIE(InfoExtractor):
 508         """Information extractor for youtube.com."""
 509
 510         _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 511         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 512         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 513         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 514         _NETRC_MACHINE = 'youtube'
 515         _available_formats = ['22', '35', '18', '5', '17', '13', None] # listed in order of priority for -b flag
 516         _video_extensions = {
 517                 '13': '3gp',
 518                 '17': 'mp4',
 519                 '18': 'mp4',
 520                 '22': 'mp4',
 521         }
 522
 523         @staticmethod
 524         def suitable(url):
 525                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 526
 527         @staticmethod
 528         def htmlentity_transform(matchobj):
 529                 """Transforms an HTML entity to a Unicode character."""
 530                 entity = matchobj.group(1)
 531
 532                 # Known non-numeric HTML entity
 533                 if entity in htmlentitydefs.name2codepoint:
 534                         return unichr(htmlentitydefs.name2codepoint[entity])
 535
 536                 # Unicode character
 537                 mobj = re.match(ur'(?u)#(x?\d+)', entity)
 538                 if mobj is not None:
 539                         numstr = mobj.group(1)
 540                         if numstr.startswith(u'x'):
 541                                 base = 16
 542                                 numstr = u'0%s' % numstr
 543                         else:
 544                                 base = 10
 545                         return unichr(long(numstr, base))
 546
 547                 # Unknown entity in name, return its literal representation
 548                 return (u'&%s;' % entity)
 549
 550         def report_lang(self):
 551                 """Report attempt to set language."""
 552                 self._downloader.to_stdout(u'[youtube] Setting language')
 553
 554         def report_login(self):
 555                 """Report attempt to log in."""
 556                 self._downloader.to_stdout(u'[youtube] Logging in')
 557
 558         def report_age_confirmation(self):
 559                 """Report attempt to confirm age."""
 560                 self._downloader.to_stdout(u'[youtube] Confirming age')
 561
 562         def report_video_info_webpage_download(self, video_id):
 563                 """Report attempt to download video info webpage."""
 564                 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
 565
 566         def report_information_extraction(self, video_id):
 567                 """Report attempt to extract video information."""
 568                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 569
 570         def report_video_url(self, video_id, video_real_url):
 571                 """Report extracted video URL."""
 572                 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
 573
 574         def report_unavailable_format(self, video_id, format):
 575                 """Report extracted video URL."""
 576                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 577
 578         def _real_initialize(self):
 579                 if self._downloader is None:
 580                         return
 581
 582                 username = None
 583                 password = None
 584                 downloader_params = self._downloader.params
 585
 586                 # Attempt to use provided username and password or .netrc data
 587                 if downloader_params.get('username', None) is not None:
 588                         username = downloader_params['username']
 589                         password = downloader_params['password']
 590                 elif downloader_params.get('usenetrc', False):
 591                         try:
 592                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 593                                 if info is not None:
 594                                         username = info[0]
 595                                         password = info[2]
 596                                 else:
 597                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 598                         except (IOError, netrc.NetrcParseError), err:
 599                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 600                                 return
 601
 602                 # Set language
 603                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 604                 try:
 605                         self.report_lang()
 606                         urllib2.urlopen(request).read()
 607                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 608                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 609                         return
 610
 611                 # No authentication to be performed
 612                 if username is None:
 613                         return
 614
 615                 # Log in
 616                 login_form = {
 617                                 'current_form': 'loginForm',
 618                                 'next':         '/',
 619                                 'action_login': 'Log In',
 620                                 'username':     username,
 621                                 'password':     password,
 622                                 }
 623                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 624                 try:
 625                         self.report_login()
 626                         login_results = urllib2.urlopen(request).read()
 627                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 628                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 629                                 return
 630                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 631                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 632                         return
 633
 634                 # Confirm age
 635                 age_form = {
 636                                 'next_url':             '/',
 637                                 'action_confirm':       'Confirm',
 638                                 }
 639                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 640                 try:
 641                         self.report_age_confirmation()
 642                         age_results = urllib2.urlopen(request).read()
 643                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 644                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 645                         return
 646
 647         def _real_extract(self, url):
 648                 # Extract video id from URL
 649                 mobj = re.match(self._VALID_URL, url)
 650                 if mobj is None:
 651                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 652                         return
 653                 video_id = mobj.group(2)
 654
 655                 # Downloader parameters
 656                 best_quality = False
 657                 format_param = None
 658                 quality_index = 0
 659                 if self._downloader is not None:
 660                         params = self._downloader.params
 661                         format_param = params.get('format', None)
 662                         if format_param == '0':
 663                                 format_param = self._available_formats[quality_index]
 664                                 best_quality = True
 665
 666                 while True:
 667                         # Extension
 668                         video_extension = self._video_extensions.get(format_param, 'flv')
 669
 670                         # Get video info
 671                         video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id
 672                         request = urllib2.Request(video_info_url, None, std_headers)
 673                         try:
 674                                 self.report_video_info_webpage_download(video_id)
 675                                 video_info_webpage = urllib2.urlopen(request).read()
 676                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 677                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 678                                 return
 679                         self.report_information_extraction(video_id)
 680
 681                         # "t" param
 682                         mobj = re.search(r'(?m)&token=([^&]+)(?:&|$)', video_info_webpage)
 683                         if mobj is None:
 684                                 # Attempt to see if YouTube has issued an error message
 685                                 mobj = re.search(r'(?m)&reason=([^&]+)(?:&|$)', video_info_webpage)
 686                                 if mobj is None:
 687                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
 688                                         stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
 689                                         stream.write(video_info_webpage)
 690                                         stream.close()
 691                                 else:
 692                                         reason = urllib.unquote_plus(mobj.group(1))
 693                                         self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
 694                                 return
 695                         token = urllib.unquote(mobj.group(1))
 696                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
 697                         if format_param is not None:
 698                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 699                         self.report_video_url(video_id, video_real_url)
 700
 701                         # uploader
 702                         mobj = re.search(r'(?m)&author=([^&]+)(?:&|$)', video_info_webpage)
 703                         if mobj is None:
 704                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 705                                 return
 706                         video_uploader = urllib.unquote(mobj.group(1))
 707
 708                         # title
 709                         mobj = re.search(r'(?m)&title=([^&]+)(?:&|$)', video_info_webpage)
 710                         if mobj is None:
 711                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 712                                 return
 713                         video_title = urllib.unquote(mobj.group(1))
 714                         video_title = video_title.decode('utf-8')
 715                         video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
 716                         video_title = video_title.replace(os.sep, u'%')
 717
 718                         # simplified title
 719                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 720                         simple_title = simple_title.strip(ur'_')
 721
 722                         try:
 723                                 # Process video information
 724                                 self._downloader.process_info({
 725                                         'id':           video_id.decode('utf-8'),
 726                                         'url':          video_real_url.decode('utf-8'),
 727                                         'uploader':     video_uploader.decode('utf-8'),
 728                                         'title':        video_title,
 729                                         'stitle':       simple_title,
 730                                         'ext':          video_extension.decode('utf-8'),
 731                                 })
 732
 733                                 return
 734
 735                         except UnavailableFormatError, err:
 736                                 if best_quality:
 737                                         if quality_index == len(self._available_formats) - 1:
 738                                                 # I don't ever expect this to happen
 739                                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 740                                                 return
 741                                         else:
 742                                                 self.report_unavailable_format(video_id, format_param)
 743                                                 quality_index += 1
 744                                                 format_param = self._available_formats[quality_index]
 745                                                 continue
 746                                 else:
 747                                         self._downloader.trouble('ERROR: format not available for video')
 748                                         return
 749
 750
 751 class MetacafeIE(InfoExtractor):
 752         """Information Extractor for metacafe.com."""
 753
 754         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 755         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 756         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 757         _youtube_ie = None
 758
 759         def __init__(self, youtube_ie, downloader=None):
 760                 InfoExtractor.__init__(self, downloader)
 761                 self._youtube_ie = youtube_ie
 762
 763         @staticmethod
 764         def suitable(url):
 765                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 766
 767         def report_disclaimer(self):
 768                 """Report disclaimer retrieval."""
 769                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 770
 771         def report_age_confirmation(self):
 772                 """Report attempt to confirm age."""
 773                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 774
 775         def report_download_webpage(self, video_id):
 776                 """Report webpage download."""
 777                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
 778
 779         def report_extraction(self, video_id):
 780                 """Report information extraction."""
 781                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
 782
 783         def _real_initialize(self):
 784                 # Retrieve disclaimer
 785                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
 786                 try:
 787                         self.report_disclaimer()
 788                         disclaimer = urllib2.urlopen(request).read()
 789                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 790                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 791                         return
 792
 793                 # Confirm age
 794                 disclaimer_form = {
 795                         'filters': '0',
 796                         'submit': "Continue - I'm over 18",
 797                         }
 798                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
 799                 try:
 800                         self.report_age_confirmation()
 801                         disclaimer = urllib2.urlopen(request).read()
 802                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 803                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 804                         return
 805
 806         def _real_extract(self, url):
 807                 # Extract id and simplified title from URL
 808                 mobj = re.match(self._VALID_URL, url)
 809                 if mobj is None:
 810                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 811                         return
 812
 813                 video_id = mobj.group(1)
 814
 815                 # Check if video comes from YouTube
 816                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 817                 if mobj2 is not None:
 818                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 819                         return
 820
 821                 simple_title = mobj.group(2).decode('utf-8')
 822                 video_extension = 'flv'
 823
 824                 # Retrieve video webpage to extract further information
 825                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 826                 try:
 827                         self.report_download_webpage(video_id)
 828                         webpage = urllib2.urlopen(request).read()
 829                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 830                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 831                         return
 832
 833                 # Extract URL, uploader and title from webpage
 834                 self.report_extraction(video_id)
 835                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 836                 if mobj is None:
 837                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 838                         return
 839                 mediaURL = urllib.unquote(mobj.group(1))
 840
 841                 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 842                 #if mobj is None:
 843                 #       self._downloader.trouble(u'ERROR: unable to extract gdaKey')
 844                 #       return
 845                 #gdaKey = mobj.group(1)
 846                 #
 847                 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 848
 849                 video_url = mediaURL
 850
 851                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 852                 if mobj is None:
 853                         self._downloader.trouble(u'ERROR: unable to extract title')
 854                         return
 855                 video_title = mobj.group(1).decode('utf-8')
 856
 857                 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
 858                 if mobj is None:
 859                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 860                         return
 861                 video_uploader = mobj.group(1)
 862
 863                 try:
 864                         # Process video information
 865                         self._downloader.process_info({
 866                                 'id':           video_id.decode('utf-8'),
 867                                 'url':          video_url.decode('utf-8'),
 868                                 'uploader':     video_uploader.decode('utf-8'),
 869                                 'title':        video_title,
 870                                 'stitle':       simple_title,
 871                                 'ext':          video_extension.decode('utf-8'),
 872                         })
 873                 except UnavailableFormatError:
 874                         self._downloader.trouble(u'ERROR: format not available for video')
 875
 876
 877 class YoutubeSearchIE(InfoExtractor):
 878         """Information Extractor for YouTube search queries."""
 879         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
 880         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
 881         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
 882         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
 883         _youtube_ie = None
 884         _max_youtube_results = 1000
 885
 886         def __init__(self, youtube_ie, downloader=None):
 887                 InfoExtractor.__init__(self, downloader)
 888                 self._youtube_ie = youtube_ie
 889
 890         @staticmethod
 891         def suitable(url):
 892                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
 893
 894         def report_download_page(self, query, pagenum):
 895                 """Report attempt to download playlist page with given number."""
 896                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
 897
 898         def _real_initialize(self):
 899                 self._youtube_ie.initialize()
 900
 901         def _real_extract(self, query):
 902                 mobj = re.match(self._VALID_QUERY, query)
 903                 if mobj is None:
 904                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
 905                         return
 906
 907                 prefix, query = query.split(':')
 908                 prefix = prefix[8:]
 909                 if prefix == '':
 910                         self._download_n_results(query, 1)
 911                         return
 912                 elif prefix == 'all':
 913                         self._download_n_results(query, self._max_youtube_results)
 914                         return
 915                 else:
 916                         try:
 917                                 n = long(prefix)
 918                                 if n <= 0:
 919                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 920                                         return
 921                                 elif n > self._max_youtube_results:
 922                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
 923                                         n = self._max_youtube_results
 924                                 self._download_n_results(query, n)
 925                                 return
 926                         except ValueError: # parsing prefix as integer fails
 927                                 self._download_n_results(query, 1)
 928                                 return
 929
 930         def _download_n_results(self, query, n):
 931                 """Downloads a specified number of results for a query"""
 932
 933                 video_ids = []
 934                 already_seen = set()
 935                 pagenum = 1
 936
 937                 while True:
 938                         self.report_download_page(query, pagenum)
 939                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
 940                         request = urllib2.Request(result_url, None, std_headers)
 941                         try:
 942                                 page = urllib2.urlopen(request).read()
 943                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 944                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
 945                                 return
 946
 947                         # Extract video identifiers
 948                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
 949                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
 950                                 if video_id not in already_seen:
 951                                         video_ids.append(video_id)
 952                                         already_seen.add(video_id)
 953                                         if len(video_ids) == n:
 954                                                 # Specified n videos reached
 955                                                 for id in video_ids:
 956                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 957                                                 return
 958
 959                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
 960                                 for id in video_ids:
 961                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 962                                 return
 963
 964                         pagenum = pagenum + 1
 965
 966 class YoutubePlaylistIE(InfoExtractor):
 967         """Information Extractor for YouTube playlists."""
 968
 969         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:view_play_list|my_playlists)\?.*?p=([^&]+).*'
 970         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
 971         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
 972         _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
 973         _youtube_ie = None
 974
 975         def __init__(self, youtube_ie, downloader=None):
 976                 InfoExtractor.__init__(self, downloader)
 977                 self._youtube_ie = youtube_ie
 978
 979         @staticmethod
 980         def suitable(url):
 981                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
 982
 983         def report_download_page(self, playlist_id, pagenum):
 984                 """Report attempt to download playlist page with given number."""
 985                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
 986
 987         def _real_initialize(self):
 988                 self._youtube_ie.initialize()
 989
 990         def _real_extract(self, url):
 991                 # Extract playlist id
 992                 mobj = re.match(self._VALID_URL, url)
 993                 if mobj is None:
 994                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
 995                         return
 996
 997                 # Download playlist pages
 998                 playlist_id = mobj.group(1)
 999                 video_ids = []
1000                 pagenum = 1
1001
1002                 while True:
1003                         self.report_download_page(playlist_id, pagenum)
1004                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1005                         try:
1006                                 page = urllib2.urlopen(request).read()
1007                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1008                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1009                                 return
1010
1011                         # Extract video identifiers
1012                         ids_in_page = []
1013                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1014                                 if mobj.group(1) not in ids_in_page:
1015                                         ids_in_page.append(mobj.group(1))
1016                         video_ids.extend(ids_in_page)
1017
1018                         if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page:
1019                                 break
1020                         pagenum = pagenum + 1
1021
1022                 for id in video_ids:
1023                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1024                 return
1025
1026 class PostProcessor(object):
1027         """Post Processor class.
1028
1029         PostProcessor objects can be added to downloaders with their
1030         add_post_processor() method. When the downloader has finished a
1031         successful download, it will take its internal chain of PostProcessors
1032         and start calling the run() method on each one of them, first with
1033         an initial argument and then with the returned value of the previous
1034         PostProcessor.
1035
1036         The chain will be stopped if one of them ever returns None or the end
1037         of the chain is reached.
1038
1039         PostProcessor objects follow a "mutual registration" process similar
1040         to InfoExtractor objects.
1041         """
1042
1043         _downloader = None
1044
1045         def __init__(self, downloader=None):
1046                 self._downloader = downloader
1047
1048         def set_downloader(self, downloader):
1049                 """Sets the downloader for this PP."""
1050                 self._downloader = downloader
1051
1052         def run(self, information):
1053                 """Run the PostProcessor.
1054
1055                 The "information" argument is a dictionary like the ones
1056                 composed by InfoExtractors. The only difference is that this
1057                 one has an extra field called "filepath" that points to the
1058                 downloaded file.
1059
1060                 When this method returns None, the postprocessing chain is
1061                 stopped. However, this method may return an information
1062                 dictionary that will be passed to the next postprocessing
1063                 object in the chain. It can be the one it received after
1064                 changing some fields.
1065
1066                 In addition, this method may raise a PostProcessingError
1067                 exception that will be taken into account by the downloader
1068                 it was called from.
1069                 """
1070                 return information # by default, do nothing
1071
1072 ### MAIN PROGRAM ###
1073 if __name__ == '__main__':
1074         try:
1075                 # Modules needed only when running the main program
1076                 import getpass
1077                 import optparse
1078
1079                 # General configuration
1080                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1081                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1082                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1083
1084                 # Parse command line
1085                 parser = optparse.OptionParser(
1086                         usage='Usage: %prog [options] url...',
1087                         version='2009.09.08',
1088                         conflict_handler='resolve',
1089                 )
1090
1091                 parser.add_option('-h', '--help',
1092                                 action='help', help='print this help text and exit')
1093                 parser.add_option('-v', '--version',
1094                                 action='version', help='print program version and exit')
1095                 parser.add_option('-i', '--ignore-errors',
1096                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1097                 parser.add_option('-r', '--rate-limit',
1098                                 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1099
1100                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1101                 authentication.add_option('-u', '--username',
1102                                 dest='username', metavar='UN', help='account username')
1103                 authentication.add_option('-p', '--password',
1104                                 dest='password', metavar='PW', help='account password')
1105                 authentication.add_option('-n', '--netrc',
1106                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1107                 parser.add_option_group(authentication)
1108
1109                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1110                 video_format.add_option('-f', '--format',
1111                                 action='store', dest='format', metavar='FMT', help='video format code')
1112                 video_format.add_option('-b', '--best-quality',
1113                                 action='store_const', dest='format', help='download the best quality video possible', const='0')
1114                 video_format.add_option('-m', '--mobile-version',
1115                                 action='store_const', dest='format', help='alias for -f 17', const='17')
1116                 video_format.add_option('-d', '--high-def',
1117                                 action='store_const', dest='format', help='alias for -f 22', const='22')
1118                 parser.add_option_group(video_format)
1119
1120                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1121                 verbosity.add_option('-q', '--quiet',
1122                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1123                 verbosity.add_option('-s', '--simulate',
1124                                 action='store_true', dest='simulate', help='do not download video', default=False)
1125                 verbosity.add_option('-g', '--get-url',
1126                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1127                 verbosity.add_option('-e', '--get-title',
1128                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1129                 parser.add_option_group(verbosity)
1130
1131                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1132                 filesystem.add_option('-t', '--title',
1133                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
1134                 filesystem.add_option('-l', '--literal',
1135                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1136                 filesystem.add_option('-o', '--output',
1137                                 dest='outtmpl', metavar='TPL', help='output filename template')
1138                 filesystem.add_option('-a', '--batch-file',
1139                                 dest='batchfile', metavar='F', help='file containing URLs to download')
1140                 filesystem.add_option('-w', '--no-overwrites',
1141                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1142                 filesystem.add_option('-c', '--continue',
1143                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1144                 parser.add_option_group(filesystem)
1145
1146                 (opts, args) = parser.parse_args()
1147
1148                 # Batch file verification
1149                 batchurls = []
1150                 if opts.batchfile is not None:
1151                         try:
1152                                 batchurls = open(opts.batchfile, 'r').readlines()
1153                                 batchurls = [x.strip() for x in batchurls]
1154                                 batchurls = [x for x in batchurls if len(x) > 0]
1155                         except IOError:
1156                                 sys.exit(u'ERROR: batch file could not be read')
1157                 all_urls = batchurls + args
1158
1159                 # Conflicting, missing and erroneous options
1160                 if len(all_urls) < 1:
1161                         parser.error(u'you must provide at least one URL')
1162                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1163                         parser.error(u'using .netrc conflicts with giving username/password')
1164                 if opts.password is not None and opts.username is None:
1165                         parser.error(u'account username missing')
1166                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1167                         parser.error(u'using output template conflicts with using title or literal title')
1168                 if opts.usetitle and opts.useliteral:
1169                         parser.error(u'using title conflicts with using literal title')
1170                 if opts.username is not None and opts.password is None:
1171                         opts.password = getpass.getpass(u'Type account password and press return:')
1172                 if opts.ratelimit is not None:
1173                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1174                         if numeric_limit is None:
1175                                 parser.error(u'invalid rate limit specified')
1176                         opts.ratelimit = numeric_limit
1177
1178                 # Information extractors
1179                 youtube_ie = YoutubeIE()
1180                 metacafe_ie = MetacafeIE(youtube_ie)
1181                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1182                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1183
1184                 # File downloader
1185                 fd = FileDownloader({
1186                         'usenetrc': opts.usenetrc,
1187                         'username': opts.username,
1188                         'password': opts.password,
1189                         'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1190                         'forceurl': opts.geturl,
1191                         'forcetitle': opts.gettitle,
1192                         'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1193                         'format': opts.format,
1194                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
1195                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1196                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1197                                 or u'%(id)s.%(ext)s'),
1198                         'ignoreerrors': opts.ignoreerrors,
1199                         'ratelimit': opts.ratelimit,
1200                         'nooverwrites': opts.nooverwrites,
1201                         'continuedl': opts.continue_dl,
1202                         })
1203                 fd.add_info_extractor(youtube_search_ie)
1204                 fd.add_info_extractor(youtube_pl_ie)
1205                 fd.add_info_extractor(metacafe_ie)
1206                 fd.add_info_extractor(youtube_ie)
1207                 retcode = fd.download(all_urls)
1208                 sys.exit(retcode)
1209
1210         except DownloadError:
1211                 sys.exit(1)
1212         except SameFileError:
1213                 sys.exit(u'ERROR: fixed output name but more than one file to download')
1214         except KeyboardInterrupt:
1215                 sys.exit(u'\nERROR: Interrupted by user')