_ Git - youtube-dl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # License: Public domain code
   6 import htmlentitydefs
   7 import httplib
   8 import locale
   9 import math
  10 import netrc
  11 import os
  12 import os.path
  13 import re
  14 import socket
  15 import string
  16 import sys
  17 import time
  18 import urllib
  19 import urllib2
  20
  21 std_headers = {
  22         'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8',
  23         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  24         'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
  25         'Accept-Language': 'en-us,en;q=0.5',
  26 }
  27
  28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  29
  30 class DownloadError(Exception):
  31         """Download Error exception.
  32
  33         This exception may be thrown by FileDownloader objects if they are not
  34         configured to continue on errors. They will contain the appropriate
  35         error message.
  36         """
  37         pass
  38
  39 class SameFileError(Exception):
  40         """Same File exception.
  41
  42         This exception will be thrown by FileDownloader objects if they detect
  43         multiple files would have to be downloaded to the same file on disk.
  44         """
  45         pass
  46
  47 class PostProcessingError(Exception):
  48         """Post Processing exception.
  49
  50         This exception may be raised by PostProcessor's .run() method to
  51         indicate an error in the postprocessing task.
  52         """
  53         pass
  54
  55 class UnavailableFormatError(Exception):
  56         """Unavailable Format exception.
  57
  58         This exception will be thrown when a video is requested
  59         in a format that is not available for that video.
  60         """
  61
  62 class FileDownloader(object):
  63         """File Downloader class.
  64
  65         File downloader objects are the ones responsible of downloading the
  66         actual video file and writing it to disk if the user has requested
  67         it, among some other tasks. In most cases there should be one per
  68         program. As, given a video URL, the downloader doesn't know how to
  69         extract all the needed information, task that InfoExtractors do, it
  70         has to pass the URL to one of them.
  71
  72         For this, file downloader objects have a method that allows
  73         InfoExtractors to be registered in a given order. When it is passed
  74         a URL, the file downloader handles it to the first InfoExtractor it
  75         finds that reports being able to handle it. The InfoExtractor extracts
  76         all the information about the video or videos the URL refers to, and
  77         asks the FileDownloader to process the video information, possibly
  78         downloading the video.
  79
  80         File downloaders accept a lot of parameters. In order not to saturate
  81         the object constructor with arguments, it receives a dictionary of
  82         options instead. These options are available through the params
  83         attribute for the InfoExtractors to use. The FileDownloader also
  84         registers itself as the downloader in charge for the InfoExtractors
  85         that are added to it, so this is a "mutual registration".
  86
  87         Available options:
  88
  89         username:       Username for authentication purposes.
  90         password:       Password for authentication purposes.
  91         usenetrc:       Use netrc for authentication instead.
  92         quiet:          Do not print messages to stdout.
  93         forceurl:       Force printing final URL.
  94         forcetitle:     Force printing title.
  95         simulate:       Do not download the video files.
  96         format:         Video format code.
  97         outtmpl:        Template for output names.
  98         ignoreerrors:   Do not stop on download errors.
  99         ratelimit:      Download speed limit, in bytes/sec.
 100         nooverwrites:   Prevent overwriting files.
 101         """
 102
 103         params = None
 104         _ies = []
 105         _pps = []
 106         _download_retcode = None
 107
 108         def __init__(self, params):
 109                 """Create a FileDownloader object with the given options."""
 110                 self._ies = []
 111                 self._pps = []
 112                 self._download_retcode = 0
 113                 self.params = params
 114
 115         @staticmethod
 116         def pmkdir(filename):
 117                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 118                 components = filename.split(os.sep)
 119                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 120                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 121                 for dir in aggregate:
 122                         if not os.path.exists(dir):
 123                                 os.mkdir(dir)
 124
 125         @staticmethod
 126         def format_bytes(bytes):
 127                 if bytes is None:
 128                         return 'N/A'
 129                 if bytes == 0:
 130                         exponent = 0
 131                 else:
 132                         exponent = long(math.log(float(bytes), 1024.0))
 133                 suffix = 'bkMGTPEZY'[exponent]
 134                 converted = float(bytes) / float(1024**exponent)
 135                 return '%.2f%s' % (converted, suffix)
 136
 137         @staticmethod
 138         def calc_percent(byte_counter, data_len):
 139                 if data_len is None:
 140                         return '---.-%'
 141                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 142
 143         @staticmethod
 144         def calc_eta(start, now, total, current):
 145                 if total is None:
 146                         return '--:--'
 147                 dif = now - start
 148                 if current == 0 or dif < 0.001: # One millisecond
 149                         return '--:--'
 150                 rate = float(current) / dif
 151                 eta = long((float(total) - float(current)) / rate)
 152                 (eta_mins, eta_secs) = divmod(eta, 60)
 153                 if eta_mins > 99:
 154                         return '--:--'
 155                 return '%02d:%02d' % (eta_mins, eta_secs)
 156
 157         @staticmethod
 158         def calc_speed(start, now, bytes):
 159                 dif = now - start
 160                 if bytes == 0 or dif < 0.001: # One millisecond
 161                         return '%10s' % '---b/s'
 162                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 163
 164         @staticmethod
 165         def best_block_size(elapsed_time, bytes):
 166                 new_min = max(bytes / 2.0, 1.0)
 167                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 168                 if elapsed_time < 0.001:
 169                         return int(new_max)
 170                 rate = bytes / elapsed_time
 171                 if rate > new_max:
 172                         return int(new_max)
 173                 if rate < new_min:
 174                         return int(new_min)
 175                 return int(rate)
 176
 177         @staticmethod
 178         def parse_bytes(bytestr):
 179                 """Parse a string indicating a byte quantity into a long integer."""
 180                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 181                 if matchobj is None:
 182                         return None
 183                 number = float(matchobj.group(1))
 184                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 185                 return long(round(number * multiplier))
 186
 187         def add_info_extractor(self, ie):
 188                 """Add an InfoExtractor object to the end of the list."""
 189                 self._ies.append(ie)
 190                 ie.set_downloader(self)
 191
 192         def add_post_processor(self, pp):
 193                 """Add a PostProcessor object to the end of the chain."""
 194                 self._pps.append(pp)
 195                 pp.set_downloader(self)
 196
 197         def to_stdout(self, message, skip_eol=False):
 198                 """Print message to stdout if not in quiet mode."""
 199                 if not self.params.get('quiet', False):
 200                         print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
 201                         sys.stdout.flush()
 202
 203         def to_stderr(self, message):
 204                 """Print message to stderr."""
 205                 print >>sys.stderr, message
 206
 207         def fixed_template(self):
 208                 """Checks if the output template is fixed."""
 209                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 210
 211         def trouble(self, message=None):
 212                 """Determine action to take when a download problem appears.
 213
 214                 Depending on if the downloader has been configured to ignore
 215                 download errors or not, this method may throw an exception or
 216                 not when errors are found, after printing the message.
 217                 """
 218                 if message is not None:
 219                         self.to_stderr(message)
 220                 if not self.params.get('ignoreerrors', False):
 221                         raise DownloadError(message)
 222                 self._download_retcode = 1
 223
 224         def slow_down(self, start_time, byte_counter):
 225                 """Sleep if the download speed is over the rate limit."""
 226                 rate_limit = self.params.get('ratelimit', None)
 227                 if rate_limit is None or byte_counter == 0:
 228                         return
 229                 now = time.time()
 230                 elapsed = now - start_time
 231                 if elapsed <= 0.0:
 232                         return
 233                 speed = float(byte_counter) / elapsed
 234                 if speed > rate_limit:
 235                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 236
 237         def report_destination(self, filename):
 238                 """Report destination filename."""
 239                 self.to_stdout(u'[download] Destination: %s' % filename)
 240
 241         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 242                 """Report download progress."""
 243                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 244                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 245
 246         def report_finish(self):
 247                 """Report download finished."""
 248                 self.to_stdout(u'')
 249
 250         def process_info(self, info_dict):
 251                 """Process a single dictionary returned by an InfoExtractor."""
 252                 # Forced printings
 253                 if self.params.get('forcetitle', False):
 254                         print info_dict['title'].encode(locale.getpreferredencoding())
 255                 if self.params.get('forceurl', False):
 256                         print info_dict['url'].encode(locale.getpreferredencoding())
 257
 258                 # Do nothing else if in simulate mode
 259                 if self.params.get('simulate', False):
 260                         return
 261
 262                 try:
 263                         filename = self.params['outtmpl'] % info_dict
 264                         self.report_destination(filename)
 265                 except (ValueError, KeyError), err:
 266                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 267                 if self.params['nooverwrites'] and os.path.exists(filename):
 268                         self.to_stderr('WARNING: file exists: %s; skipping' % filename)
 269                         return
 270
 271                 try:
 272                         self.pmkdir(filename)
 273                 except (OSError, IOError), err:
 274                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 275                         return
 276
 277                 try:
 278                         outstream = open(filename, 'wb')
 279                 except (OSError, IOError), err:
 280                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 281                         return
 282
 283                 try:
 284                         self._do_download(outstream, info_dict['url'])
 285                         outstream.close()
 286                 except (OSError, IOError), err:
 287                         os.remove(filename)
 288                         raise UnavailableFormatError
 289                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 290                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 291                         return
 292
 293                 try:
 294                         self.post_process(filename, info_dict)
 295                 except (PostProcessingError), err:
 296                         self.trouble('ERROR: postprocessing: %s' % str(err))
 297                         return
 298
 299         def download(self, url_list):
 300                 """Download a given list of URLs."""
 301                 if len(url_list) > 1 and self.fixed_template():
 302                         raise SameFileError(self.params['outtmpl'])
 303
 304                 for url in url_list:
 305                         suitable_found = False
 306                         for ie in self._ies:
 307                                 # Go to next InfoExtractor if not suitable
 308                                 if not ie.suitable(url):
 309                                         continue
 310
 311                                 # Suitable InfoExtractor found
 312                                 suitable_found = True
 313
 314                                 # Extract information from URL and process it
 315                                 ie.extract(url)
 316
 317                                 # Suitable InfoExtractor had been found; go to next URL
 318                                 break
 319
 320                         if not suitable_found:
 321                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 322
 323                 return self._download_retcode
 324
 325         def post_process(self, filename, ie_info):
 326                 """Run the postprocessing chain on the given file."""
 327                 info = dict(ie_info)
 328                 info['filepath'] = filename
 329                 for pp in self._pps:
 330                         info = pp.run(info)
 331                         if info is None:
 332                                 break
 333
 334         def _do_download(self, stream, url):
 335                 request = urllib2.Request(url, None, std_headers)
 336                 data = urllib2.urlopen(request)
 337                 data_len = data.info().get('Content-length', None)
 338                 data_len_str = self.format_bytes(data_len)
 339                 byte_counter = 0
 340                 block_size = 1024
 341                 start = time.time()
 342                 while True:
 343                         # Progress message
 344                         percent_str = self.calc_percent(byte_counter, data_len)
 345                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 346                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 347                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 348
 349                         # Download and write
 350                         before = time.time()
 351                         data_block = data.read(block_size)
 352                         after = time.time()
 353                         data_block_len = len(data_block)
 354                         if data_block_len == 0:
 355                                 break
 356                         byte_counter += data_block_len
 357                         stream.write(data_block)
 358                         block_size = self.best_block_size(after - before, data_block_len)
 359
 360                         # Apply rate limit
 361                         self.slow_down(start, byte_counter)
 362
 363                 self.report_finish()
 364                 if data_len is not None and str(byte_counter) != data_len:
 365                         raise ValueError('Content too short: %s/%s bytes' % (byte_counter, data_len))
 366
 367 class InfoExtractor(object):
 368         """Information Extractor class.
 369
 370         Information extractors are the classes that, given a URL, extract
 371         information from the video (or videos) the URL refers to. This
 372         information includes the real video URL, the video title and simplified
 373         title, author and others. The information is stored in a dictionary
 374         which is then passed to the FileDownloader. The FileDownloader
 375         processes this information possibly downloading the video to the file
 376         system, among other possible outcomes. The dictionaries must include
 377         the following fields:
 378
 379         id:             Video identifier.
 380         url:            Final video URL.
 381         uploader:       Nickname of the video uploader.
 382         title:          Literal title.
 383         stitle:         Simplified title.
 384         ext:            Video filename extension.
 385
 386         Subclasses of this one should re-define the _real_initialize() and
 387         _real_extract() methods, as well as the suitable() static method.
 388         Probably, they should also be instantiated and added to the main
 389         downloader.
 390         """
 391
 392         _ready = False
 393         _downloader = None
 394
 395         def __init__(self, downloader=None):
 396                 """Constructor. Receives an optional downloader."""
 397                 self._ready = False
 398                 self.set_downloader(downloader)
 399
 400         @staticmethod
 401         def suitable(url):
 402                 """Receives a URL and returns True if suitable for this IE."""
 403                 return False
 404
 405         def initialize(self):
 406                 """Initializes an instance (authentication, etc)."""
 407                 if not self._ready:
 408                         self._real_initialize()
 409                         self._ready = True
 410
 411         def extract(self, url):
 412                 """Extracts URL information and returns it in list of dicts."""
 413                 self.initialize()
 414                 return self._real_extract(url)
 415
 416         def set_downloader(self, downloader):
 417                 """Sets the downloader for this IE."""
 418                 self._downloader = downloader
 419
 420         def _real_initialize(self):
 421                 """Real initialization process. Redefine in subclasses."""
 422                 pass
 423
 424         def _real_extract(self, url):
 425                 """Real extraction process. Redefine in subclasses."""
 426                 pass
 427
 428 class YoutubeIE(InfoExtractor):
 429         """Information extractor for youtube.com."""
 430
 431         _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 432         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 433         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 434         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 435         _NETRC_MACHINE = 'youtube'
 436         _available_formats = ['22', '35', '18', '17', '13'] # listed in order of priority for -b flag
 437         _video_extensions = {
 438                 '13': '3gp',
 439                 '17': 'mp4',
 440                 '18': 'mp4',
 441                 '22': 'mp4',
 442         }
 443
 444         @staticmethod
 445         def suitable(url):
 446                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 447
 448         @staticmethod
 449         def htmlentity_transform(matchobj):
 450                 """Transforms an HTML entity to a Unicode character."""
 451                 entity = matchobj.group(1)
 452
 453                 # Known non-numeric HTML entity
 454                 if entity in htmlentitydefs.name2codepoint:
 455                         return unichr(htmlentitydefs.name2codepoint[entity])
 456
 457                 # Unicode character
 458                 mobj = re.match(ur'(?u)#(x?\d+)', entity)
 459                 if mobj is not None:
 460                         numstr = mobj.group(1)
 461                         if numstr.startswith(u'x'):
 462                                 base = 16
 463                                 numstr = u'0%s' % numstr
 464                         else:
 465                                 base = 10
 466                         return unichr(long(numstr, base))
 467
 468                 # Unknown entity in name, return its literal representation
 469                 return (u'&%s;' % entity)
 470
 471         def report_lang(self):
 472                 """Report attempt to set language."""
 473                 self._downloader.to_stdout(u'[youtube] Setting language')
 474
 475         def report_login(self):
 476                 """Report attempt to log in."""
 477                 self._downloader.to_stdout(u'[youtube] Logging in')
 478
 479         def report_age_confirmation(self):
 480                 """Report attempt to confirm age."""
 481                 self._downloader.to_stdout(u'[youtube] Confirming age')
 482
 483         def report_webpage_download(self, video_id):
 484                 """Report attempt to download webpage."""
 485                 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
 486
 487         def report_information_extraction(self, video_id):
 488                 """Report attempt to extract video information."""
 489                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 490
 491         def report_video_url(self, video_id, video_real_url):
 492                 """Report extracted video URL."""
 493                 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
 494
 495         def report_unavailable_format(self, video_id, format):
 496                 """Report extracted video URL."""
 497                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 498
 499         def _real_initialize(self):
 500                 if self._downloader is None:
 501                         return
 502
 503                 username = None
 504                 password = None
 505                 downloader_params = self._downloader.params
 506
 507                 # Attempt to use provided username and password or .netrc data
 508                 if downloader_params.get('username', None) is not None:
 509                         username = downloader_params['username']
 510                         password = downloader_params['password']
 511                 elif downloader_params.get('usenetrc', False):
 512                         try:
 513                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 514                                 if info is not None:
 515                                         username = info[0]
 516                                         password = info[2]
 517                                 else:
 518                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 519                         except (IOError, netrc.NetrcParseError), err:
 520                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 521                                 return
 522
 523                 # Set language
 524                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 525                 try:
 526                         self.report_lang()
 527                         urllib2.urlopen(request).read()
 528                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 529                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 530                         return
 531
 532                 # No authentication to be performed
 533                 if username is None:
 534                         return
 535
 536                 # Log in
 537                 login_form = {
 538                                 'current_form': 'loginForm',
 539                                 'next':         '/',
 540                                 'action_login': 'Log In',
 541                                 'username':     username,
 542                                 'password':     password,
 543                                 }
 544                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 545                 try:
 546                         self.report_login()
 547                         login_results = urllib2.urlopen(request).read()
 548                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 549                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 550                                 return
 551                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 552                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 553                         return
 554
 555                 # Confirm age
 556                 age_form = {
 557                                 'next_url':             '/',
 558                                 'action_confirm':       'Confirm',
 559                                 }
 560                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 561                 try:
 562                         self.report_age_confirmation()
 563                         age_results = urllib2.urlopen(request).read()
 564                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 565                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 566                         return
 567
 568         def _real_extract(self, url):
 569                 # Extract video id from URL
 570                 mobj = re.match(self._VALID_URL, url)
 571                 if mobj is None:
 572                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 573                         return
 574                 video_id = mobj.group(2)
 575
 576                 # Downloader parameters
 577                 best_quality = False
 578                 format_param = None
 579                 quality_index = 0
 580                 if self._downloader is not None:
 581                         params = self._downloader.params
 582                         format_param = params.get('format', None)
 583                         if format_param == '0':
 584                                 format_param = self._available_formats[quality_index]
 585                                 best_quality = True
 586
 587                 while True:
 588                         try:
 589                                 # Extension
 590                                 video_extension = self._video_extensions.get(format_param, 'flv')
 591
 592                                 # Normalize URL, including format
 593                                 normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
 594                                 if format_param is not None:
 595                                         normalized_url = '%s&fmt=%s' % (normalized_url, format_param)
 596                                 request = urllib2.Request(normalized_url, None, std_headers)
 597                                 try:
 598                                         self.report_webpage_download(video_id)
 599                                         video_webpage = urllib2.urlopen(request).read()
 600                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 601                                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 602                                         return
 603                                 self.report_information_extraction(video_id)
 604
 605                                 # "t" param
 606                                 mobj = re.search(r', "t": "([^"]+)"', video_webpage)
 607                                 if mobj is None:
 608                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
 609                                         return
 610                                 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
 611                                 if format_param is not None:
 612                                         video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 613                                 self.report_video_url(video_id, video_real_url)
 614
 615                                 # uploader
 616                                 mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
 617                                 if mobj is None:
 618                                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 619                                         return
 620                                 video_uploader = mobj.group(1)
 621
 622                                 # title
 623                                 mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
 624                                 if mobj is None:
 625                                         self._downloader.trouble(u'ERROR: unable to extract video title')
 626                                         return
 627                                 video_title = mobj.group(1).decode('utf-8')
 628                                 video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
 629                                 video_title = video_title.replace(os.sep, u'%')
 630
 631                                 # simplified title
 632                                 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 633                                 simple_title = simple_title.strip(ur'_')
 634
 635                                 # Process video information
 636                                 self._downloader.process_info({
 637                                         'id':           video_id.decode('utf-8'),
 638                                         'url':          video_real_url.decode('utf-8'),
 639                                         'uploader':     video_uploader.decode('utf-8'),
 640                                         'title':        video_title,
 641                                         'stitle':       simple_title,
 642                                         'ext':          video_extension.decode('utf-8'),
 643                                 })
 644
 645                                 return
 646
 647                         except UnavailableFormatError, err:
 648                                 if best_quality:
 649                                         if quality_index == len(self._available_formats) - 1:
 650                                                 # I don't ever expect this to happen
 651                                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 652                                                 return
 653                                         else:
 654                                                 self.report_unavailable_format(video_id, format_param)
 655                                                 quality_index += 1
 656                                                 format_param = self._available_formats[quality_index]
 657                                                 continue
 658                                 else:
 659                                         self._downloader.trouble('ERROR: format not available for video')
 660                                         return
 661
 662
 663 class MetacafeIE(InfoExtractor):
 664         """Information Extractor for metacafe.com."""
 665
 666         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 667         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 668         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 669         _youtube_ie = None
 670
 671         def __init__(self, youtube_ie, downloader=None):
 672                 InfoExtractor.__init__(self, downloader)
 673                 self._youtube_ie = youtube_ie
 674
 675         @staticmethod
 676         def suitable(url):
 677                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 678
 679         def report_disclaimer(self):
 680                 """Report disclaimer retrieval."""
 681                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 682
 683         def report_age_confirmation(self):
 684                 """Report attempt to confirm age."""
 685                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 686
 687         def report_download_webpage(self, video_id):
 688                 """Report webpage download."""
 689                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
 690
 691         def report_extraction(self, video_id):
 692                 """Report information extraction."""
 693                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
 694
 695         def _real_initialize(self):
 696                 # Retrieve disclaimer
 697                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
 698                 try:
 699                         self.report_disclaimer()
 700                         disclaimer = urllib2.urlopen(request).read()
 701                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 702                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 703                         return
 704
 705                 # Confirm age
 706                 disclaimer_form = {
 707                         'filters': '0',
 708                         'submit': "Continue - I'm over 18",
 709                         }
 710                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
 711                 try:
 712                         self.report_age_confirmation()
 713                         disclaimer = urllib2.urlopen(request).read()
 714                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 715                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 716                         return
 717
 718         def _real_extract(self, url):
 719                 # Extract id and simplified title from URL
 720                 mobj = re.match(self._VALID_URL, url)
 721                 if mobj is None:
 722                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 723                         return
 724
 725                 video_id = mobj.group(1)
 726
 727                 # Check if video comes from YouTube
 728                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 729                 if mobj2 is not None:
 730                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 731                         return
 732
 733                 simple_title = mobj.group(2).decode('utf-8')
 734                 video_extension = 'flv'
 735
 736                 # Retrieve video webpage to extract further information
 737                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 738                 try:
 739                         self.report_download_webpage(video_id)
 740                         webpage = urllib2.urlopen(request).read()
 741                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 742                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 743                         return
 744
 745                 # Extract URL, uploader and title from webpage
 746                 self.report_extraction(video_id)
 747                 mobj = re.search(r'(?m)&mediaURL=(http.*?\.flv)', webpage)
 748                 if mobj is None:
 749                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 750                         return
 751                 mediaURL = urllib.unquote(mobj.group(1))
 752
 753                 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 754                 if mobj is None:
 755                         self._downloader.trouble(u'ERROR: unable to extract gdaKey')
 756                         return
 757                 gdaKey = mobj.group(1)
 758
 759                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 760
 761                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 762                 if mobj is None:
 763                         self._downloader.trouble(u'ERROR: unable to extract title')
 764                         return
 765                 video_title = mobj.group(1).decode('utf-8')
 766
 767                 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
 768                 if mobj is None:
 769                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 770                         return
 771                 video_uploader = mobj.group(1)
 772
 773                 try:
 774                         # Process video information
 775                         self._downloader.process_info({
 776                                 'id':           video_id.decode('utf-8'),
 777                                 'url':          video_url.decode('utf-8'),
 778                                 'uploader':     video_uploader.decode('utf-8'),
 779                                 'title':        video_title,
 780                                 'stitle':       simple_title,
 781                                 'ext':          video_extension.decode('utf-8'),
 782                         })
 783                 except UnavailableFormatError:
 784                         self._downloader.trouble(u'ERROR: format not available for video')
 785
 786
 787 class YoutubeSearchIE(InfoExtractor):
 788         """Information Extractor for YouTube search queries."""
 789         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
 790         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
 791         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
 792         _MORE_PAGES_INDICATOR = r'>Next</a>'
 793         _youtube_ie = None
 794         _max_youtube_results = 1000
 795
 796         def __init__(self, youtube_ie, downloader=None):
 797                 InfoExtractor.__init__(self, downloader)
 798                 self._youtube_ie = youtube_ie
 799
 800         @staticmethod
 801         def suitable(url):
 802                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
 803
 804         def report_download_page(self, query, pagenum):
 805                 """Report attempt to download playlist page with given number."""
 806                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
 807
 808         def _real_initialize(self):
 809                 self._youtube_ie.initialize()
 810
 811         def _real_extract(self, query):
 812                 mobj = re.match(self._VALID_QUERY, query)
 813                 if mobj is None:
 814                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
 815                         return
 816
 817                 prefix, query = query.split(':')
 818                 prefix = prefix[8:]
 819                 if prefix == '':
 820                         self._download_n_results(query, 1)
 821                         return
 822                 elif prefix == 'all':
 823                         self._download_n_results(query, self._max_youtube_results)
 824                         return
 825                 else:
 826                         try:
 827                                 n = int(prefix)
 828                                 if n <= 0:
 829                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 830                                         return
 831                                 elif n > self._max_youtube_results:
 832                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
 833                                         n = self._max_youtube_results
 834                                 self._download_n_results(query, n)
 835                                 return
 836                         except ValueError: # parsing prefix as int fails
 837                                 self._download_n_results(query, 1)
 838                                 return
 839
 840         def _download_n_results(self, query, n):
 841                 """Downloads a specified number of results for a query"""
 842
 843                 video_ids = []
 844                 already_seen = set()
 845                 pagenum = 1
 846
 847                 while True:
 848                         self.report_download_page(query, pagenum)
 849                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
 850                         request = urllib2.Request(result_url, None, std_headers)
 851                         try:
 852                                 page = urllib2.urlopen(request).read()
 853                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 854                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
 855                                 return
 856
 857                         # Extract video identifiers
 858                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
 859                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
 860                                 if video_id not in already_seen:
 861                                         video_ids.append(video_id)
 862                                         already_seen.add(video_id)
 863                                         if len(video_ids) == n:
 864                                                 # Specified n videos reached
 865                                                 for id in video_ids:
 866                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 867                                                 return
 868
 869                         if self._MORE_PAGES_INDICATOR not in page:
 870                                 for id in video_ids:
 871                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 872                                 return
 873
 874                         pagenum = pagenum + 1
 875
 876 class YoutubePlaylistIE(InfoExtractor):
 877         """Information Extractor for YouTube playlists."""
 878
 879         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)'
 880         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
 881         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
 882         _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&amp;page=%s'
 883         _youtube_ie = None
 884
 885         def __init__(self, youtube_ie, downloader=None):
 886                 InfoExtractor.__init__(self, downloader)
 887                 self._youtube_ie = youtube_ie
 888
 889         @staticmethod
 890         def suitable(url):
 891                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
 892
 893         def report_download_page(self, playlist_id, pagenum):
 894                 """Report attempt to download playlist page with given number."""
 895                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
 896
 897         def _real_initialize(self):
 898                 self._youtube_ie.initialize()
 899
 900         def _real_extract(self, url):
 901                 # Extract playlist id
 902                 mobj = re.match(self._VALID_URL, url)
 903                 if mobj is None:
 904                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
 905                         return
 906
 907                 # Download playlist pages
 908                 playlist_id = mobj.group(1)
 909                 video_ids = []
 910                 pagenum = 1
 911
 912                 while True:
 913                         self.report_download_page(playlist_id, pagenum)
 914                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
 915                         try:
 916                                 page = urllib2.urlopen(request).read()
 917                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 918                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
 919                                 return
 920
 921                         # Extract video identifiers
 922                         ids_in_page = []
 923                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
 924                                 if mobj.group(1) not in ids_in_page:
 925                                         ids_in_page.append(mobj.group(1))
 926                         video_ids.extend(ids_in_page)
 927
 928                         if (self._MORE_PAGES_INDICATOR % (playlist_id, pagenum + 1)) not in page:
 929                                 break
 930                         pagenum = pagenum + 1
 931
 932                 for id in video_ids:
 933                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 934                 return
 935
 936 class PostProcessor(object):
 937         """Post Processor class.
 938
 939         PostProcessor objects can be added to downloaders with their
 940         add_post_processor() method. When the downloader has finished a
 941         successful download, it will take its internal chain of PostProcessors
 942         and start calling the run() method on each one of them, first with
 943         an initial argument and then with the returned value of the previous
 944         PostProcessor.
 945
 946         The chain will be stopped if one of them ever returns None or the end
 947         of the chain is reached.
 948
 949         PostProcessor objects follow a "mutual registration" process similar
 950         to InfoExtractor objects.
 951         """
 952
 953         _downloader = None
 954
 955         def __init__(self, downloader=None):
 956                 self._downloader = downloader
 957
 958         def set_downloader(self, downloader):
 959                 """Sets the downloader for this PP."""
 960                 self._downloader = downloader
 961
 962         def run(self, information):
 963                 """Run the PostProcessor.
 964
 965                 The "information" argument is a dictionary like the ones
 966                 composed by InfoExtractors. The only difference is that this
 967                 one has an extra field called "filepath" that points to the
 968                 downloaded file.
 969
 970                 When this method returns None, the postprocessing chain is
 971                 stopped. However, this method may return an information
 972                 dictionary that will be passed to the next postprocessing
 973                 object in the chain. It can be the one it received after
 974                 changing some fields.
 975
 976                 In addition, this method may raise a PostProcessingError
 977                 exception that will be taken into account by the downloader
 978                 it was called from.
 979                 """
 980                 return information # by default, do nothing
 981
 982 ### MAIN PROGRAM ###
 983 if __name__ == '__main__':
 984         try:
 985                 # Modules needed only when running the main program
 986                 import getpass
 987                 import optparse
 988
 989                 # General configuration
 990                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
 991                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
 992                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
 993
 994                 # Parse command line
 995                 parser = optparse.OptionParser(
 996                         usage='Usage: %prog [options] url...',
 997                         version='INTERNAL',
 998                         conflict_handler='resolve',
 999                 )
1000
1001                 parser.add_option('-h', '--help',
1002                                 action='help', help='print this help text and exit')
1003                 parser.add_option('-v', '--version',
1004                                 action='version', help='print program version and exit')
1005                 parser.add_option('-i', '--ignore-errors',
1006                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1007                 parser.add_option('-r', '--rate-limit',
1008                                 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1009
1010                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1011                 authentication.add_option('-u', '--username',
1012                                 dest='username', metavar='UN', help='account username')
1013                 authentication.add_option('-p', '--password',
1014                                 dest='password', metavar='PW', help='account password')
1015                 authentication.add_option('-n', '--netrc',
1016                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1017                 parser.add_option_group(authentication)
1018
1019                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1020                 video_format.add_option('-f', '--format',
1021                                 action='append', dest='format', metavar='FMT', help='video format code')
1022                 video_format.add_option('-b', '--best-quality',
1023                                 action='append_const', dest='format', help='download the best quality video possible', const='0')
1024                 video_format.add_option('-m', '--mobile-version',
1025                                 action='append_const', dest='format', help='alias for -f 17', const='17')
1026                 video_format.add_option('-d', '--high-def',
1027                                 action='append_const', dest='format', help='alias for -f 22', const='22')
1028                 parser.add_option_group(video_format)
1029
1030                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1031                 verbosity.add_option('-q', '--quiet',
1032                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1033                 verbosity.add_option('-s', '--simulate',
1034                                 action='store_true', dest='simulate', help='do not download video', default=False)
1035                 verbosity.add_option('-g', '--get-url',
1036                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1037                 verbosity.add_option('-e', '--get-title',
1038                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1039                 parser.add_option_group(verbosity)
1040
1041                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1042                 filesystem.add_option('-t', '--title',
1043                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
1044                 filesystem.add_option('-l', '--literal',
1045                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1046                 filesystem.add_option('-o', '--output',
1047                                 dest='outtmpl', metavar='TPL', help='output filename template')
1048                 filesystem.add_option('-a', '--batch-file',
1049                                 dest='batchfile', metavar='F', help='file containing URLs to download')
1050                 filesystem.add_option('-w', '--no-overwrites',
1051                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1052                 parser.add_option_group(filesystem)
1053
1054                 (opts, args) = parser.parse_args()
1055
1056                 # Batch file verification
1057                 batchurls = []
1058                 if opts.batchfile is not None:
1059                         try:
1060                                 batchurls = open(opts.batchfile, 'r').readlines()
1061                                 batchurls = [x.strip() for x in batchurls]
1062                                 batchurls = [x for x in batchurls if len(x) > 0]
1063                         except IOError:
1064                                 sys.exit(u'ERROR: batch file could not be read')
1065                 all_urls = batchurls + args
1066
1067                 # Conflicting, missing and erroneous options
1068                 if len(all_urls) < 1:
1069                         parser.error(u'you must provide at least one URL')
1070                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1071                         parser.error(u'using .netrc conflicts with giving username/password')
1072                 if opts.password is not None and opts.username is None:
1073                         parser.error(u'account username missing')
1074                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1075                         parser.error(u'using output template conflicts with using title or literal title')
1076                 if opts.usetitle and opts.useliteral:
1077                         parser.error(u'using title conflicts with using literal title')
1078                 if opts.username is not None and opts.password is None:
1079                         opts.password = getpass.getpass(u'Type account password and press return:')
1080                 if opts.ratelimit is not None:
1081                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1082                         if numeric_limit is None:
1083                                 parser.error(u'invalid rate limit specified')
1084                         opts.ratelimit = numeric_limit
1085                 if opts.format is not None and len(opts.format) > 1:
1086                         parser.error(u'pass at most one of the video format option flags (-f, -b, -m, -d)')
1087                 if opts.format is None:
1088                         real_format = None
1089                 else:
1090                         real_format = opts.format[0]
1091
1092
1093                 # Information extractors
1094                 youtube_ie = YoutubeIE()
1095                 metacafe_ie = MetacafeIE(youtube_ie)
1096                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1097                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1098
1099                 # File downloader
1100                 fd = FileDownloader({
1101                         'usenetrc': opts.usenetrc,
1102                         'username': opts.username,
1103                         'password': opts.password,
1104                         'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1105                         'forceurl': opts.geturl,
1106                         'forcetitle': opts.gettitle,
1107                         'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1108                         'format': real_format,
1109                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
1110                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1111                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1112                                 or u'%(id)s.%(ext)s'),
1113                         'ignoreerrors': opts.ignoreerrors,
1114                         'ratelimit': opts.ratelimit,
1115                         'nooverwrites': opts.nooverwrites,
1116                         })
1117                 fd.add_info_extractor(youtube_search_ie)
1118                 fd.add_info_extractor(youtube_pl_ie)
1119                 fd.add_info_extractor(metacafe_ie)
1120                 fd.add_info_extractor(youtube_ie)
1121                 retcode = fd.download(all_urls)
1122                 sys.exit(retcode)
1123
1124         except DownloadError:
1125                 sys.exit(1)
1126         except SameFileError:
1127                 sys.exit(u'ERROR: fixed output name but more than one file to download')
1128         except KeyboardInterrupt:
1129                 sys.exit(u'\nERROR: Interrupted by user')