_ Git - youtube-dl/blob - youtube_dl/InfoExtractors.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import datetime
   5 import HTMLParser
   6 import httplib
   7 import netrc
   8 import os
   9 import re
  10 import socket
  11 import time
  12 import urllib
  13 import urllib2
  14 import email.utils
  15
  16 try:
  17         import cStringIO as StringIO
  18 except ImportError:
  19         import StringIO
  20
  21 # parse_qs was moved from the cgi module to the urlparse module recently.
  22 try:
  23         from urlparse import parse_qs
  24 except ImportError:
  25         from cgi import parse_qs
  26
  27 try:
  28         import lxml.etree
  29 except ImportError:
  30         pass # Handled below
  31
  32 try:
  33         import xml.etree.ElementTree
  34 except ImportError: # Python<2.5: Not officially supported, but let it slip
  35         warnings.warn('xml.etree.ElementTree support is missing. Consider upgrading to Python >= 2.5 if you get related errors.')
  36
  37 from Utils import *
  38
  39
  40 class InfoExtractor(object):
  41         """Information Extractor class.
  42
  43         Information extractors are the classes that, given a URL, extract
  44         information from the video (or videos) the URL refers to. This
  45         information includes the real video URL, the video title and simplified
  46         title, author and others. The information is stored in a dictionary
  47         which is then passed to the FileDownloader. The FileDownloader
  48         processes this information possibly downloading the video to the file
  49         system, among other possible outcomes. The dictionaries must include
  50         the following fields:
  51
  52         id:             Video identifier.
  53         url:            Final video URL.
  54         uploader:       Nickname of the video uploader.
  55         title:          Literal title.
  56         stitle:         Simplified title.
  57         ext:            Video filename extension.
  58         format:         Video format.
  59         player_url:     SWF Player URL (may be None).
  60
  61         The following fields are optional. Their primary purpose is to allow
  62         youtube-dl to serve as the backend for a video search function, such
  63         as the one in youtube2mp3.  They are only used when their respective
  64         forced printing functions are called:
  65
  66         thumbnail:      Full URL to a video thumbnail image.
  67         description:    One-line video description.
  68
  69         Subclasses of this one should re-define the _real_initialize() and
  70         _real_extract() methods and define a _VALID_URL regexp.
  71         Probably, they should also be added to the list of extractors.
  72         """
  73
  74         _ready = False
  75         _downloader = None
  76
  77         def __init__(self, downloader=None):
  78                 """Constructor. Receives an optional downloader."""
  79                 self._ready = False
  80                 self.set_downloader(downloader)
  81
  82         def suitable(self, url):
  83                 """Receives a URL and returns True if suitable for this IE."""
  84                 return re.match(self._VALID_URL, url) is not None
  85
  86         def initialize(self):
  87                 """Initializes an instance (authentication, etc)."""
  88                 if not self._ready:
  89                         self._real_initialize()
  90                         self._ready = True
  91
  92         def extract(self, url):
  93                 """Extracts URL information and returns it in list of dicts."""
  94                 self.initialize()
  95                 return self._real_extract(url)
  96
  97         def set_downloader(self, downloader):
  98                 """Sets the downloader for this IE."""
  99                 self._downloader = downloader
 100
 101         def _real_initialize(self):
 102                 """Real initialization process. Redefine in subclasses."""
 103                 pass
 104
 105         def _real_extract(self, url):
 106                 """Real extraction process. Redefine in subclasses."""
 107                 pass
 108
 109
 110 class YoutubeIE(InfoExtractor):
 111         """Information extractor for youtube.com."""
 112
 113         _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
 114         _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 115         _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
 116         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 117         _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 118         _NETRC_MACHINE = 'youtube'
 119         # Listed in order of quality
 120         _available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
 121         _available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
 122         _video_extensions = {
 123                 '13': '3gp',
 124                 '17': 'mp4',
 125                 '18': 'mp4',
 126                 '22': 'mp4',
 127                 '37': 'mp4',
 128                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 129                 '43': 'webm',
 130                 '44': 'webm',
 131                 '45': 'webm',
 132         }
 133         _video_dimensions = {
 134                 '5': '240x400',
 135                 '6': '???',
 136                 '13': '???',
 137                 '17': '144x176',
 138                 '18': '360x640',
 139                 '22': '720x1280',
 140                 '34': '360x640',
 141                 '35': '480x854',
 142                 '37': '1080x1920',
 143                 '38': '3072x4096',
 144                 '43': '360x640',
 145                 '44': '480x854',
 146                 '45': '720x1280',
 147         }
 148         IE_NAME = u'youtube'
 149
 150         def report_lang(self):
 151                 """Report attempt to set language."""
 152                 self._downloader.to_screen(u'[youtube] Setting language')
 153
 154         def report_login(self):
 155                 """Report attempt to log in."""
 156                 self._downloader.to_screen(u'[youtube] Logging in')
 157
 158         def report_age_confirmation(self):
 159                 """Report attempt to confirm age."""
 160                 self._downloader.to_screen(u'[youtube] Confirming age')
 161
 162         def report_video_webpage_download(self, video_id):
 163                 """Report attempt to download video webpage."""
 164                 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
 165
 166         def report_video_info_webpage_download(self, video_id):
 167                 """Report attempt to download video info webpage."""
 168                 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
 169
 170         def report_video_subtitles_download(self, video_id):
 171                 """Report attempt to download video info webpage."""
 172                 self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
 173
 174         def report_information_extraction(self, video_id):
 175                 """Report attempt to extract video information."""
 176                 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
 177
 178         def report_unavailable_format(self, video_id, format):
 179                 """Report extracted video URL."""
 180                 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
 181
 182         def report_rtmp_download(self):
 183                 """Indicate the download will use the RTMP protocol."""
 184                 self._downloader.to_screen(u'[youtube] RTMP download detected')
 185
 186         def _closed_captions_xml_to_srt(self, xml_string):
 187                 srt = ''
 188                 texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
 189                 # TODO parse xml instead of regex
 190                 for n, (start, dur_tag, dur, caption) in enumerate(texts):
 191                         if not dur: dur = '4'
 192                         start = float(start)
 193                         end = start + float(dur)
 194                         start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
 195                         end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
 196                         caption = re.sub(ur'(?u)&(.+?);', htmlentity_transform, caption)
 197                         caption = re.sub(ur'(?u)&(.+?);', htmlentity_transform, caption) # double cycle, inentional
 198                         srt += str(n) + '\n'
 199                         srt += start + ' --> ' + end + '\n'
 200                         srt += caption + '\n\n'
 201                 return srt
 202
 203         def _print_formats(self, formats):
 204                 print 'Available formats:'
 205                 for x in formats:
 206                         print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))
 207
 208         def _real_initialize(self):
 209                 if self._downloader is None:
 210                         return
 211
 212                 username = None
 213                 password = None
 214                 downloader_params = self._downloader.params
 215
 216                 # Attempt to use provided username and password or .netrc data
 217                 if downloader_params.get('username', None) is not None:
 218                         username = downloader_params['username']
 219                         password = downloader_params['password']
 220                 elif downloader_params.get('usenetrc', False):
 221                         try:
 222                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 223                                 if info is not None:
 224                                         username = info[0]
 225                                         password = info[2]
 226                                 else:
 227                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 228                         except (IOError, netrc.NetrcParseError), err:
 229                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 230                                 return
 231
 232                 # Set language
 233                 request = urllib2.Request(self._LANG_URL)
 234                 try:
 235                         self.report_lang()
 236                         urllib2.urlopen(request).read()
 237                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 238                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 239                         return
 240
 241                 # No authentication to be performed
 242                 if username is None:
 243                         return
 244
 245                 # Log in
 246                 login_form = {
 247                                 'current_form': 'loginForm',
 248                                 'next':         '/',
 249                                 'action_login': 'Log In',
 250                                 'username':     username,
 251                                 'password':     password,
 252                                 }
 253                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
 254                 try:
 255                         self.report_login()
 256                         login_results = urllib2.urlopen(request).read()
 257                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 258                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 259                                 return
 260                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 261                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 262                         return
 263
 264                 # Confirm age
 265                 age_form = {
 266                                 'next_url':             '/',
 267                                 'action_confirm':       'Confirm',
 268                                 }
 269                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
 270                 try:
 271                         self.report_age_confirmation()
 272                         age_results = urllib2.urlopen(request).read()
 273                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 274                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 275                         return
 276
 277         def _real_extract(self, url):
 278                 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
 279                 mobj = re.search(self._NEXT_URL_RE, url)
 280                 if mobj:
 281                         url = 'http://www.youtube.com/' + urllib.unquote(mobj.group(1)).lstrip('/')
 282
 283                 # Extract video id from URL
 284                 mobj = re.match(self._VALID_URL, url)
 285                 if mobj is None:
 286                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 287                         return
 288                 video_id = mobj.group(2)
 289
 290                 # Get video webpage
 291                 self.report_video_webpage_download(video_id)
 292                 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
 293                 try:
 294                         video_webpage = urllib2.urlopen(request).read()
 295                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 296                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 297                         return
 298
 299                 # Attempt to extract SWF player URL
 300                 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
 301                 if mobj is not None:
 302                         player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
 303                 else:
 304                         player_url = None
 305
 306                 # Get video info
 307                 self.report_video_info_webpage_download(video_id)
 308                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
 309                         video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
 310                                         % (video_id, el_type))
 311                         request = urllib2.Request(video_info_url)
 312                         try:
 313                                 video_info_webpage = urllib2.urlopen(request).read()
 314                                 video_info = parse_qs(video_info_webpage)
 315                                 if 'token' in video_info:
 316                                         break
 317                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 318                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 319                                 return
 320                 if 'token' not in video_info:
 321                         if 'reason' in video_info:
 322                                 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
 323                         else:
 324                                 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
 325                         return
 326
 327                 # Start extracting information
 328                 self.report_information_extraction(video_id)
 329
 330                 # uploader
 331                 if 'author' not in video_info:
 332                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 333                         return
 334                 video_uploader = urllib.unquote_plus(video_info['author'][0])
 335
 336                 # title
 337                 if 'title' not in video_info:
 338                         self._downloader.trouble(u'ERROR: unable to extract video title')
 339                         return
 340                 video_title = urllib.unquote_plus(video_info['title'][0])
 341                 video_title = video_title.decode('utf-8')
 342                 video_title = sanitize_title(video_title)
 343
 344                 # simplified title
 345                 simple_title = simplify_title(video_title)
 346
 347                 # thumbnail image
 348                 if 'thumbnail_url' not in video_info:
 349                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
 350                         video_thumbnail = ''
 351                 else:   # don't panic if we can't find it
 352                         video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
 353
 354                 # upload date
 355                 upload_date = u'NA'
 356                 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
 357                 if mobj is not None:
 358                         upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
 359                         format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
 360                         for expression in format_expressions:
 361                                 try:
 362                                         upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
 363                                 except:
 364                                         pass
 365
 366                 # description
 367                 try:
 368                         lxml.etree
 369                 except NameError:
 370                         video_description = u'No description available.'
 371                         mobj = re.search(r'<meta name="description" content="(.*?)">', video_webpage)
 372                         if mobj is not None:
 373                                 video_description = mobj.group(1).decode('utf-8')
 374                 else:
 375                         html_parser = lxml.etree.HTMLParser(encoding='utf-8')
 376                         vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
 377                         video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()'))
 378                         # TODO use another parser
 379
 380                 # closed captions
 381                 video_subtitles = None
 382                 if self._downloader.params.get('writesubtitles', False):
 383                         self.report_video_subtitles_download(video_id)
 384                         request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
 385                         try:
 386                                 srt_list = urllib2.urlopen(request).read()
 387                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 388                                 self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
 389                         else:
 390                                 srt_lang_list = re.findall(r'lang_code="([\w\-]+)"', srt_list)
 391                                 if srt_lang_list:
 392                                         if self._downloader.params.get('subtitleslang', False):
 393                                                 srt_lang = self._downloader.params.get('subtitleslang')
 394                                         elif 'en' in srt_lang_list:
 395                                                 srt_lang = 'en'
 396                                         else:
 397                                                 srt_lang = srt_lang_list[0]
 398                                         if not srt_lang in srt_lang_list:
 399                                                 self._downloader.trouble(u'WARNING: no closed captions found in the specified language')
 400                                         else:
 401                                                 request = urllib2.Request('http://video.google.com/timedtext?hl=en&lang=%s&v=%s' % (srt_lang, video_id))
 402                                                 try:
 403                                                         srt_xml = urllib2.urlopen(request).read()
 404                                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 405                                                         self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
 406                                                 else:
 407                                                         video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
 408                                 else:
 409                                         self._downloader.trouble(u'WARNING: video has no closed captions')
 410
 411                 # token
 412                 video_token = urllib.unquote_plus(video_info['token'][0])
 413
 414                 # Decide which formats to download
 415                 req_format = self._downloader.params.get('format', None)
 416
 417                 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 418                         self.report_rtmp_download()
 419                         video_url_list = [(None, video_info['conn'][0])]
 420                 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
 421                         url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
 422                         url_data = [parse_qs(uds) for uds in url_data_strs]
 423                         url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
 424                         url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
 425
 426                         format_limit = self._downloader.params.get('format_limit', None)
 427                         available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
 428                         if format_limit is not None and format_limit in available_formats:
 429                                 format_list = available_formats[available_formats.index(format_limit):]
 430                         else:
 431                                 format_list = available_formats
 432                         existing_formats = [x for x in format_list if x in url_map]
 433                         if len(existing_formats) == 0:
 434                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 435                                 return
 436                         if self._downloader.params.get('listformats', None):
 437                                 self._print_formats(existing_formats)
 438                                 return
 439                         if req_format is None or req_format == 'best':
 440                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
 441                         elif req_format == 'worst':
 442                                 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
 443                         elif req_format in ('-1', 'all'):
 444                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
 445                         else:
 446                                 # Specific formats. We pick the first in a slash-delimeted sequence.
 447                                 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
 448                                 req_formats = req_format.split('/')
 449                                 video_url_list = None
 450                                 for rf in req_formats:
 451                                         if rf in url_map:
 452                                                 video_url_list = [(rf, url_map[rf])]
 453                                                 break
 454                                 if video_url_list is None:
 455                                         self._downloader.trouble(u'ERROR: requested format not available')
 456                                         return
 457                 else:
 458                         self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
 459                         return
 460
 461                 for format_param, video_real_url in video_url_list:
 462                         # At this point we have a new video
 463                         self._downloader.increment_downloads()
 464
 465                         # Extension
 466                         video_extension = self._video_extensions.get(format_param, 'flv')
 467
 468                         try:
 469                                 # Process video information
 470                                 self._downloader.process_info({
 471                                         'id':           video_id.decode('utf-8'),
 472                                         'url':          video_real_url.decode('utf-8'),
 473                                         'uploader':     video_uploader.decode('utf-8'),
 474                                         'upload_date':  upload_date,
 475                                         'title':        video_title,
 476                                         'stitle':       simple_title,
 477                                         'ext':          video_extension.decode('utf-8'),
 478                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
 479                                         'thumbnail':    video_thumbnail.decode('utf-8'),
 480                                         'description':  video_description,
 481                                         'player_url':   player_url,
 482                                         'subtitles':    video_subtitles
 483                                 })
 484                         except UnavailableVideoError, err:
 485                                 self._downloader.trouble(u'\nERROR: unable to download video')
 486
 487
 488 class MetacafeIE(InfoExtractor):
 489         """Information Extractor for metacafe.com."""
 490
 491         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 492         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 493         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 494         _youtube_ie = None
 495         IE_NAME = u'metacafe'
 496
 497         def __init__(self, youtube_ie, downloader=None):
 498                 InfoExtractor.__init__(self, downloader)
 499                 self._youtube_ie = youtube_ie
 500
 501         def report_disclaimer(self):
 502                 """Report disclaimer retrieval."""
 503                 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
 504
 505         def report_age_confirmation(self):
 506                 """Report attempt to confirm age."""
 507                 self._downloader.to_screen(u'[metacafe] Confirming age')
 508
 509         def report_download_webpage(self, video_id):
 510                 """Report webpage download."""
 511                 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
 512
 513         def report_extraction(self, video_id):
 514                 """Report information extraction."""
 515                 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
 516
 517         def _real_initialize(self):
 518                 # Retrieve disclaimer
 519                 request = urllib2.Request(self._DISCLAIMER)
 520                 try:
 521                         self.report_disclaimer()
 522                         disclaimer = urllib2.urlopen(request).read()
 523                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 524                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 525                         return
 526
 527                 # Confirm age
 528                 disclaimer_form = {
 529                         'filters': '0',
 530                         'submit': "Continue - I'm over 18",
 531                         }
 532                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
 533                 try:
 534                         self.report_age_confirmation()
 535                         disclaimer = urllib2.urlopen(request).read()
 536                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 537                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 538                         return
 539
 540         def _real_extract(self, url):
 541                 # Extract id and simplified title from URL
 542                 mobj = re.match(self._VALID_URL, url)
 543                 if mobj is None:
 544                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 545                         return
 546
 547                 video_id = mobj.group(1)
 548
 549                 # Check if video comes from YouTube
 550                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 551                 if mobj2 is not None:
 552                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 553                         return
 554
 555                 # At this point we have a new video
 556                 self._downloader.increment_downloads()
 557
 558                 simple_title = mobj.group(2).decode('utf-8')
 559
 560                 # Retrieve video webpage to extract further information
 561                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 562                 try:
 563                         self.report_download_webpage(video_id)
 564                         webpage = urllib2.urlopen(request).read()
 565                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 566                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 567                         return
 568
 569                 # Extract URL, uploader and title from webpage
 570                 self.report_extraction(video_id)
 571                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 572                 if mobj is not None:
 573                         mediaURL = urllib.unquote(mobj.group(1))
 574                         video_extension = mediaURL[-3:]
 575
 576                         # Extract gdaKey if available
 577                         mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 578                         if mobj is None:
 579                                 video_url = mediaURL
 580                         else:
 581                                 gdaKey = mobj.group(1)
 582                                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 583                 else:
 584                         mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
 585                         if mobj is None:
 586                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 587                                 return
 588                         vardict = parse_qs(mobj.group(1))
 589                         if 'mediaData' not in vardict:
 590                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 591                                 return
 592                         mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
 593                         if mobj is None:
 594                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 595                                 return
 596                         mediaURL = mobj.group(1).replace('\\/', '/')
 597                         video_extension = mediaURL[-3:]
 598                         video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
 599
 600                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 601                 if mobj is None:
 602                         self._downloader.trouble(u'ERROR: unable to extract title')
 603                         return
 604                 video_title = mobj.group(1).decode('utf-8')
 605                 video_title = sanitize_title(video_title)
 606
 607                 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
 608                 if mobj is None:
 609                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 610                         return
 611                 video_uploader = mobj.group(1)
 612
 613                 try:
 614                         # Process video information
 615                         self._downloader.process_info({
 616                                 'id':           video_id.decode('utf-8'),
 617                                 'url':          video_url.decode('utf-8'),
 618                                 'uploader':     video_uploader.decode('utf-8'),
 619                                 'upload_date':  u'NA',
 620                                 'title':        video_title,
 621                                 'stitle':       simple_title,
 622                                 'ext':          video_extension.decode('utf-8'),
 623                                 'format':       u'NA',
 624                                 'player_url':   None,
 625                         })
 626                 except UnavailableVideoError:
 627                         self._downloader.trouble(u'\nERROR: unable to download video')
 628
 629
 630 class DailymotionIE(InfoExtractor):
 631         """Information Extractor for Dailymotion"""
 632
 633         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
 634         IE_NAME = u'dailymotion'
 635
 636         def __init__(self, downloader=None):
 637                 InfoExtractor.__init__(self, downloader)
 638
 639         def report_download_webpage(self, video_id):
 640                 """Report webpage download."""
 641                 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
 642
 643         def report_extraction(self, video_id):
 644                 """Report information extraction."""
 645                 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
 646
 647         def _real_extract(self, url):
 648                 # Extract id and simplified title from URL
 649                 mobj = re.match(self._VALID_URL, url)
 650                 if mobj is None:
 651                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 652                         return
 653
 654                 # At this point we have a new video
 655                 self._downloader.increment_downloads()
 656                 video_id = mobj.group(1)
 657
 658                 video_extension = 'flv'
 659
 660                 # Retrieve video webpage to extract further information
 661                 request = urllib2.Request(url)
 662                 request.add_header('Cookie', 'family_filter=off')
 663                 try:
 664                         self.report_download_webpage(video_id)
 665                         webpage = urllib2.urlopen(request).read()
 666                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 667                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 668                         return
 669
 670                 # Extract URL, uploader and title from webpage
 671                 self.report_extraction(video_id)
 672                 mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage)
 673                 if mobj is None:
 674                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 675                         return
 676                 sequence = urllib.unquote(mobj.group(1))
 677                 mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence)
 678                 if mobj is None:
 679                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 680                         return
 681                 mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '')
 682
 683                 # if needed add http://www.dailymotion.com/ if relative URL
 684
 685                 video_url = mediaURL
 686
 687                 mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
 688                 if mobj is None:
 689                         self._downloader.trouble(u'ERROR: unable to extract title')
 690                         return
 691                 video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
 692                 video_title = sanitize_title(video_title)
 693                 simple_title = simplify_title(video_title)
 694
 695                 mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
 696                 if mobj is None:
 697                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 698                         return
 699                 video_uploader = mobj.group(1)
 700
 701                 try:
 702                         # Process video information
 703                         self._downloader.process_info({
 704                                 'id':           video_id.decode('utf-8'),
 705                                 'url':          video_url.decode('utf-8'),
 706                                 'uploader':     video_uploader.decode('utf-8'),
 707                                 'upload_date':  u'NA',
 708                                 'title':        video_title,
 709                                 'stitle':       simple_title,
 710                                 'ext':          video_extension.decode('utf-8'),
 711                                 'format':       u'NA',
 712                                 'player_url':   None,
 713                         })
 714                 except UnavailableVideoError:
 715                         self._downloader.trouble(u'\nERROR: unable to download video')
 716
 717
 718 class GoogleIE(InfoExtractor):
 719         """Information extractor for video.google.com."""
 720
 721         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
 722         IE_NAME = u'video.google'
 723
 724         def __init__(self, downloader=None):
 725                 InfoExtractor.__init__(self, downloader)
 726
 727         def report_download_webpage(self, video_id):
 728                 """Report webpage download."""
 729                 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
 730
 731         def report_extraction(self, video_id):
 732                 """Report information extraction."""
 733                 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
 734
 735         def _real_extract(self, url):
 736                 # Extract id from URL
 737                 mobj = re.match(self._VALID_URL, url)
 738                 if mobj is None:
 739                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 740                         return
 741
 742                 # At this point we have a new video
 743                 self._downloader.increment_downloads()
 744                 video_id = mobj.group(1)
 745
 746                 video_extension = 'mp4'
 747
 748                 # Retrieve video webpage to extract further information
 749                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
 750                 try:
 751                         self.report_download_webpage(video_id)
 752                         webpage = urllib2.urlopen(request).read()
 753                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 754                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 755                         return
 756
 757                 # Extract URL, uploader, and title from webpage
 758                 self.report_extraction(video_id)
 759                 mobj = re.search(r"download_url:'([^']+)'", webpage)
 760                 if mobj is None:
 761                         video_extension = 'flv'
 762                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
 763                 if mobj is None:
 764                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 765                         return
 766                 mediaURL = urllib.unquote(mobj.group(1))
 767                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
 768                 mediaURL = mediaURL.replace('\\x26', '\x26')
 769
 770                 video_url = mediaURL
 771
 772                 mobj = re.search(r'<title>(.*)</title>', webpage)
 773                 if mobj is None:
 774                         self._downloader.trouble(u'ERROR: unable to extract title')
 775                         return
 776                 video_title = mobj.group(1).decode('utf-8')
 777                 video_title = sanitize_title(video_title)
 778                 simple_title = simplify_title(video_title)
 779
 780                 # Extract video description
 781                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
 782                 if mobj is None:
 783                         self._downloader.trouble(u'ERROR: unable to extract video description')
 784                         return
 785                 video_description = mobj.group(1).decode('utf-8')
 786                 if not video_description:
 787                         video_description = 'No description available.'
 788
 789                 # Extract video thumbnail
 790                 if self._downloader.params.get('forcethumbnail', False):
 791                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
 792                         try:
 793                                 webpage = urllib2.urlopen(request).read()
 794                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 795                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 796                                 return
 797                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
 798                         if mobj is None:
 799                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
 800                                 return
 801                         video_thumbnail = mobj.group(1)
 802                 else:   # we need something to pass to process_info
 803                         video_thumbnail = ''
 804
 805                 try:
 806                         # Process video information
 807                         self._downloader.process_info({
 808                                 'id':           video_id.decode('utf-8'),
 809                                 'url':          video_url.decode('utf-8'),
 810                                 'uploader':     u'NA',
 811                                 'upload_date':  u'NA',
 812                                 'title':        video_title,
 813                                 'stitle':       simple_title,
 814                                 'ext':          video_extension.decode('utf-8'),
 815                                 'format':       u'NA',
 816                                 'player_url':   None,
 817                         })
 818                 except UnavailableVideoError:
 819                         self._downloader.trouble(u'\nERROR: unable to download video')
 820
 821
 822 class PhotobucketIE(InfoExtractor):
 823         """Information extractor for photobucket.com."""
 824
 825         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
 826         IE_NAME = u'photobucket'
 827
 828         def __init__(self, downloader=None):
 829                 InfoExtractor.__init__(self, downloader)
 830
 831         def report_download_webpage(self, video_id):
 832                 """Report webpage download."""
 833                 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
 834
 835         def report_extraction(self, video_id):
 836                 """Report information extraction."""
 837                 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
 838
 839         def _real_extract(self, url):
 840                 # Extract id from URL
 841                 mobj = re.match(self._VALID_URL, url)
 842                 if mobj is None:
 843                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 844                         return
 845
 846                 # At this point we have a new video
 847                 self._downloader.increment_downloads()
 848                 video_id = mobj.group(1)
 849
 850                 video_extension = 'flv'
 851
 852                 # Retrieve video webpage to extract further information
 853                 request = urllib2.Request(url)
 854                 try:
 855                         self.report_download_webpage(video_id)
 856                         webpage = urllib2.urlopen(request).read()
 857                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 858                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 859                         return
 860
 861                 # Extract URL, uploader, and title from webpage
 862                 self.report_extraction(video_id)
 863                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
 864                 if mobj is None:
 865                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 866                         return
 867                 mediaURL = urllib.unquote(mobj.group(1))
 868
 869                 video_url = mediaURL
 870
 871                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
 872                 if mobj is None:
 873                         self._downloader.trouble(u'ERROR: unable to extract title')
 874                         return
 875                 video_title = mobj.group(1).decode('utf-8')
 876                 video_title = sanitize_title(video_title)
 877                 simple_title = simplify_title(video_title)
 878
 879                 video_uploader = mobj.group(2).decode('utf-8')
 880
 881                 try:
 882                         # Process video information
 883                         self._downloader.process_info({
 884                                 'id':           video_id.decode('utf-8'),
 885                                 'url':          video_url.decode('utf-8'),
 886                                 'uploader':     video_uploader,
 887                                 'upload_date':  u'NA',
 888                                 'title':        video_title,
 889                                 'stitle':       simple_title,
 890                                 'ext':          video_extension.decode('utf-8'),
 891                                 'format':       u'NA',
 892                                 'player_url':   None,
 893                         })
 894                 except UnavailableVideoError:
 895                         self._downloader.trouble(u'\nERROR: unable to download video')
 896
 897
 898 class YahooIE(InfoExtractor):
 899         """Information extractor for video.yahoo.com."""
 900
 901         # _VALID_URL matches all Yahoo! Video URLs
 902         # _VPAGE_URL matches only the extractable '/watch/' URLs
 903         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
 904         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
 905         IE_NAME = u'video.yahoo'
 906
 907         def __init__(self, downloader=None):
 908                 InfoExtractor.__init__(self, downloader)
 909
 910         def report_download_webpage(self, video_id):
 911                 """Report webpage download."""
 912                 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
 913
 914         def report_extraction(self, video_id):
 915                 """Report information extraction."""
 916                 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
 917
 918         def _real_extract(self, url, new_video=True):
 919                 # Extract ID from URL
 920                 mobj = re.match(self._VALID_URL, url)
 921                 if mobj is None:
 922                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 923                         return
 924
 925                 # At this point we have a new video
 926                 self._downloader.increment_downloads()
 927                 video_id = mobj.group(2)
 928                 video_extension = 'flv'
 929
 930                 # Rewrite valid but non-extractable URLs as
 931                 # extractable English language /watch/ URLs
 932                 if re.match(self._VPAGE_URL, url) is None:
 933                         request = urllib2.Request(url)
 934                         try:
 935                                 webpage = urllib2.urlopen(request).read()
 936                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 937                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 938                                 return
 939
 940                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
 941                         if mobj is None:
 942                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
 943                                 return
 944                         yahoo_id = mobj.group(1)
 945
 946                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
 947                         if mobj is None:
 948                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
 949                                 return
 950                         yahoo_vid = mobj.group(1)
 951
 952                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
 953                         return self._real_extract(url, new_video=False)
 954
 955                 # Retrieve video webpage to extract further information
 956                 request = urllib2.Request(url)
 957                 try:
 958                         self.report_download_webpage(video_id)
 959                         webpage = urllib2.urlopen(request).read()
 960                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 961                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 962                         return
 963
 964                 # Extract uploader and title from webpage
 965                 self.report_extraction(video_id)
 966                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
 967                 if mobj is None:
 968                         self._downloader.trouble(u'ERROR: unable to extract video title')
 969                         return
 970                 video_title = mobj.group(1).decode('utf-8')
 971                 simple_title = simplify_title(video_title)
 972
 973                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
 974                 if mobj is None:
 975                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
 976                         return
 977                 video_uploader = mobj.group(1).decode('utf-8')
 978
 979                 # Extract video thumbnail
 980                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
 981                 if mobj is None:
 982                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
 983                         return
 984                 video_thumbnail = mobj.group(1).decode('utf-8')
 985
 986                 # Extract video description
 987                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
 988                 if mobj is None:
 989                         self._downloader.trouble(u'ERROR: unable to extract video description')
 990                         return
 991                 video_description = mobj.group(1).decode('utf-8')
 992                 if not video_description:
 993                         video_description = 'No description available.'
 994
 995                 # Extract video height and width
 996                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
 997                 if mobj is None:
 998                         self._downloader.trouble(u'ERROR: unable to extract video height')
 999                         return
1000                 yv_video_height = mobj.group(1)
1001
1002                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1003                 if mobj is None:
1004                         self._downloader.trouble(u'ERROR: unable to extract video width')
1005                         return
1006                 yv_video_width = mobj.group(1)
1007
1008                 # Retrieve video playlist to extract media URL
1009                 # I'm not completely sure what all these options are, but we
1010                 # seem to need most of them, otherwise the server sends a 401.
1011                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1012                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1013                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1014                                 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1015                                 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1016                 try:
1017                         self.report_download_webpage(video_id)
1018                         webpage = urllib2.urlopen(request).read()
1019                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1020                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1021                         return
1022
1023                 # Extract media URL from playlist XML
1024                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1025                 if mobj is None:
1026                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1027                         return
1028                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1029                 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1030
1031                 try:
1032                         # Process video information
1033                         self._downloader.process_info({
1034                                 'id':           video_id.decode('utf-8'),
1035                                 'url':          video_url,
1036                                 'uploader':     video_uploader,
1037                                 'upload_date':  u'NA',
1038                                 'title':        video_title,
1039                                 'stitle':       simple_title,
1040                                 'ext':          video_extension.decode('utf-8'),
1041                                 'thumbnail':    video_thumbnail.decode('utf-8'),
1042                                 'description':  video_description,
1043                                 'thumbnail':    video_thumbnail,
1044                                 'player_url':   None,
1045                         })
1046                 except UnavailableVideoError:
1047                         self._downloader.trouble(u'\nERROR: unable to download video')
1048
1049
1050 class VimeoIE(InfoExtractor):
1051         """Information extractor for vimeo.com."""
1052
1053         # _VALID_URL matches Vimeo URLs
1054         _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
1055         IE_NAME = u'vimeo'
1056
1057         def __init__(self, downloader=None):
1058                 InfoExtractor.__init__(self, downloader)
1059
1060         def report_download_webpage(self, video_id):
1061                 """Report webpage download."""
1062                 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1063
1064         def report_extraction(self, video_id):
1065                 """Report information extraction."""
1066                 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1067
1068         def _real_extract(self, url, new_video=True):
1069                 # Extract ID from URL
1070                 mobj = re.match(self._VALID_URL, url)
1071                 if mobj is None:
1072                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1073                         return
1074
1075                 # At this point we have a new video
1076                 self._downloader.increment_downloads()
1077                 video_id = mobj.group(1)
1078
1079                 # Retrieve video webpage to extract further information
1080                 request = urllib2.Request(url, None, std_headers)
1081                 try:
1082                         self.report_download_webpage(video_id)
1083                         webpage = urllib2.urlopen(request).read()
1084                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1085                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1086                         return
1087
1088                 # Now we begin extracting as much information as we can from what we
1089                 # retrieved. First we extract the information common to all extractors,
1090                 # and latter we extract those that are Vimeo specific.
1091                 self.report_extraction(video_id)
1092
1093                 # Extract the config JSON
1094                 config = webpage.split(' = {config:')[1].split(',assets:')[0]
1095                 try:
1096                         config = json.loads(config)
1097                 except:
1098                         self._downloader.trouble(u'ERROR: unable to extract info section')
1099                         return
1100
1101                 # Extract title
1102                 video_title = config["video"]["title"]
1103                 simple_title = simplify_title(video_title)
1104
1105                 # Extract uploader
1106                 video_uploader = config["video"]["owner"]["name"]
1107
1108                 # Extract video thumbnail
1109                 video_thumbnail = config["video"]["thumbnail"]
1110
1111                 # Extract video description
1112                 try:
1113                         lxml.etree
1114                 except NameError:
1115                         video_description = u'No description available.'
1116                         mobj = re.search(r'<meta name="description" content="(.*?)" />', webpage, re.MULTILINE)
1117                         if mobj is not None:
1118                                 video_description = mobj.group(1)
1119                 else:
1120                         html_parser = lxml.etree.HTMLParser()
1121                         vwebpage_doc = lxml.etree.parse(StringIO.StringIO(webpage), html_parser)
1122                         video_description = u''.join(vwebpage_doc.xpath('id("description")//text()')).strip()
1123                         # TODO use another parser
1124
1125                 # Extract upload date
1126                 video_upload_date = u'NA'
1127                 mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
1128                 if mobj is not None:
1129                         video_upload_date = mobj.group(1)
1130
1131                 # Vimeo specific: extract request signature and timestamp
1132                 sig = config['request']['signature']
1133                 timestamp = config['request']['timestamp']
1134
1135                 # Vimeo specific: extract video codec and quality information
1136                 # TODO bind to format param
1137                 codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
1138                 for codec in codecs:
1139                         if codec[0] in config["video"]["files"]:
1140                                 video_codec = codec[0]
1141                                 video_extension = codec[1]
1142                                 if 'hd' in config["video"]["files"][codec[0]]: quality = 'hd'
1143                                 else: quality = 'sd'
1144                                 break
1145                 else:
1146                         self._downloader.trouble(u'ERROR: no known codec found')
1147                         return
1148
1149                 video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
1150                                         %(video_id, sig, timestamp, quality, video_codec.upper())
1151
1152                 try:
1153                         # Process video information
1154                         self._downloader.process_info({
1155                                 'id':           video_id,
1156                                 'url':          video_url,
1157                                 'uploader':     video_uploader,
1158                                 'upload_date':  video_upload_date,
1159                                 'title':        video_title,
1160                                 'stitle':       simple_title,
1161                                 'ext':          video_extension,
1162                                 'thumbnail':    video_thumbnail,
1163                                 'description':  video_description,
1164                                 'player_url':   None,
1165                         })
1166                 except UnavailableVideoError:
1167                         self._downloader.trouble(u'ERROR: unable to download video')
1168
1169
1170 class GenericIE(InfoExtractor):
1171         """Generic last-resort information extractor."""
1172
1173         _VALID_URL = r'.*'
1174         IE_NAME = u'generic'
1175
1176         def __init__(self, downloader=None):
1177                 InfoExtractor.__init__(self, downloader)
1178
1179         def report_download_webpage(self, video_id):
1180                 """Report webpage download."""
1181                 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1182                 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1183
1184         def report_extraction(self, video_id):
1185                 """Report information extraction."""
1186                 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1187
1188         def report_following_redirect(self, new_url):
1189                 """Report information extraction."""
1190                 self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
1191
1192         def _test_redirect(self, url):
1193                 """Check if it is a redirect, like url shorteners, in case restart chain."""
1194                 class HeadRequest(urllib2.Request):
1195                         def get_method(self):
1196                                 return "HEAD"
1197
1198                 class HEADRedirectHandler(urllib2.HTTPRedirectHandler):
1199                         """
1200                         Subclass the HTTPRedirectHandler to make it use our
1201                         HeadRequest also on the redirected URL
1202                         """
1203                         def redirect_request(self, req, fp, code, msg, headers, newurl):
1204                                 if code in (301, 302, 303, 307):
1205                                     newurl = newurl.replace(' ', '%20')
1206                                     newheaders = dict((k,v) for k,v in req.headers.items()
1207                                                       if k.lower() not in ("content-length", "content-type"))
1208                                     return HeadRequest(newurl,
1209                                                        headers=newheaders,
1210                                                        origin_req_host=req.get_origin_req_host(),
1211                                                        unverifiable=True)
1212                                 else:
1213                                     raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
1214
1215                 class HTTPMethodFallback(urllib2.BaseHandler):
1216                         """
1217                         Fallback to GET if HEAD is not allowed (405 HTTP error)
1218                         """
1219                         def http_error_405(self, req, fp, code, msg, headers):
1220                                 fp.read()
1221                                 fp.close()
1222
1223                                 newheaders = dict((k,v) for k,v in req.headers.items()
1224                                                   if k.lower() not in ("content-length", "content-type"))
1225                                 return self.parent.open(urllib2.Request(req.get_full_url(),
1226                                                                  headers=newheaders,
1227                                                                  origin_req_host=req.get_origin_req_host(),
1228                                                                  unverifiable=True))
1229
1230                 # Build our opener
1231                 opener = urllib2.OpenerDirector()
1232                 for handler in [urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler,
1233                                         HTTPMethodFallback, HEADRedirectHandler,
1234                                         urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler]:
1235                         opener.add_handler(handler())
1236
1237                 response = opener.open(HeadRequest(url))
1238                 new_url = response.geturl()
1239
1240                 if url == new_url: return False
1241
1242                 self.report_following_redirect(new_url)
1243                 self._downloader.download([new_url])
1244                 return True
1245
1246         def _real_extract(self, url):
1247                 if self._test_redirect(url): return
1248
1249                 # At this point we have a new video
1250                 self._downloader.increment_downloads()
1251
1252                 video_id = url.split('/')[-1]
1253                 request = urllib2.Request(url)
1254                 try:
1255                         self.report_download_webpage(video_id)
1256                         webpage = urllib2.urlopen(request).read()
1257                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1258                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1259                         return
1260                 except ValueError, err:
1261                         # since this is the last-resort InfoExtractor, if
1262                         # this error is thrown, it'll be thrown here
1263                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1264                         return
1265
1266                 self.report_extraction(video_id)
1267                 # Start with something easy: JW Player in SWFObject
1268                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1269                 if mobj is None:
1270                         # Broaden the search a little bit
1271                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1272                 if mobj is None:
1273                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1274                         return
1275
1276                 # It's possible that one of the regexes
1277                 # matched, but returned an empty group:
1278                 if mobj.group(1) is None:
1279                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1280                         return
1281
1282                 video_url = urllib.unquote(mobj.group(1))
1283                 video_id = os.path.basename(video_url)
1284
1285                 # here's a fun little line of code for you:
1286                 video_extension = os.path.splitext(video_id)[1][1:]
1287                 video_id = os.path.splitext(video_id)[0]
1288
1289                 # it's tempting to parse this further, but you would
1290                 # have to take into account all the variations like
1291                 #   Video Title - Site Name
1292                 #   Site Name | Video Title
1293                 #   Video Title - Tagline | Site Name
1294                 # and so on and so forth; it's just not practical
1295                 mobj = re.search(r'<title>(.*)</title>', webpage)
1296                 if mobj is None:
1297                         self._downloader.trouble(u'ERROR: unable to extract title')
1298                         return
1299                 video_title = mobj.group(1).decode('utf-8')
1300                 video_title = sanitize_title(video_title)
1301                 simple_title = simplify_title(video_title)
1302
1303                 # video uploader is domain name
1304                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1305                 if mobj is None:
1306                         self._downloader.trouble(u'ERROR: unable to extract title')
1307                         return
1308                 video_uploader = mobj.group(1).decode('utf-8')
1309
1310                 try:
1311                         # Process video information
1312                         self._downloader.process_info({
1313                                 'id':           video_id.decode('utf-8'),
1314                                 'url':          video_url.decode('utf-8'),
1315                                 'uploader':     video_uploader,
1316                                 'upload_date':  u'NA',
1317                                 'title':        video_title,
1318                                 'stitle':       simple_title,
1319                                 'ext':          video_extension.decode('utf-8'),
1320                                 'format':       u'NA',
1321                                 'player_url':   None,
1322                         })
1323                 except UnavailableVideoError, err:
1324                         self._downloader.trouble(u'\nERROR: unable to download video')
1325
1326
1327 class YoutubeSearchIE(InfoExtractor):
1328         """Information Extractor for YouTube search queries."""
1329         _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
1330         _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1331         _youtube_ie = None
1332         _max_youtube_results = 1000
1333         IE_NAME = u'youtube:search'
1334
1335         def __init__(self, youtube_ie, downloader=None):
1336                 InfoExtractor.__init__(self, downloader)
1337                 self._youtube_ie = youtube_ie
1338
1339         def report_download_page(self, query, pagenum):
1340                 """Report attempt to download playlist page with given number."""
1341                 query = query.decode(preferredencoding())
1342                 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1343
1344         def _real_initialize(self):
1345                 self._youtube_ie.initialize()
1346
1347         def _real_extract(self, query):
1348                 mobj = re.match(self._VALID_URL, query)
1349                 if mobj is None:
1350                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1351                         return
1352
1353                 prefix, query = query.split(':')
1354                 prefix = prefix[8:]
1355                 query = query.encode('utf-8')
1356                 if prefix == '':
1357                         self._download_n_results(query, 1)
1358                         return
1359                 elif prefix == 'all':
1360                         self._download_n_results(query, self._max_youtube_results)
1361                         return
1362                 else:
1363                         try:
1364                                 n = long(prefix)
1365                                 if n <= 0:
1366                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1367                                         return
1368                                 elif n > self._max_youtube_results:
1369                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1370                                         n = self._max_youtube_results
1371                                 self._download_n_results(query, n)
1372                                 return
1373                         except ValueError: # parsing prefix as integer fails
1374                                 self._download_n_results(query, 1)
1375                                 return
1376
1377         def _download_n_results(self, query, n):
1378                 """Downloads a specified number of results for a query"""
1379
1380                 video_ids = []
1381                 pagenum = 0
1382                 limit = n
1383
1384                 while (50 * pagenum) < limit:
1385                         self.report_download_page(query, pagenum+1)
1386                         result_url = self._API_URL % (urllib.quote_plus(query), (50*pagenum)+1)
1387                         request = urllib2.Request(result_url)
1388                         try:
1389                                 data = urllib2.urlopen(request).read()
1390                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1391                                 self._downloader.trouble(u'ERROR: unable to download API page: %s' % str(err))
1392                                 return
1393                         api_response = json.loads(data)['data']
1394
1395                         new_ids = list(video['id'] for video in api_response['items'])
1396                         video_ids += new_ids
1397
1398                         limit = min(n, api_response['totalItems'])
1399                         pagenum += 1
1400
1401                 if len(video_ids) > n:
1402                         video_ids = video_ids[:n]
1403                 for id in video_ids:
1404                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1405                 return
1406
1407
1408 class GoogleSearchIE(InfoExtractor):
1409         """Information Extractor for Google Video search queries."""
1410         _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
1411         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1412         _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
1413         _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
1414         _google_ie = None
1415         _max_google_results = 1000
1416         IE_NAME = u'video.google:search'
1417
1418         def __init__(self, google_ie, downloader=None):
1419                 InfoExtractor.__init__(self, downloader)
1420                 self._google_ie = google_ie
1421
1422         def report_download_page(self, query, pagenum):
1423                 """Report attempt to download playlist page with given number."""
1424                 query = query.decode(preferredencoding())
1425                 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1426
1427         def _real_initialize(self):
1428                 self._google_ie.initialize()
1429
1430         def _real_extract(self, query):
1431                 mobj = re.match(self._VALID_URL, query)
1432                 if mobj is None:
1433                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1434                         return
1435
1436                 prefix, query = query.split(':')
1437                 prefix = prefix[8:]
1438                 query = query.encode('utf-8')
1439                 if prefix == '':
1440                         self._download_n_results(query, 1)
1441                         return
1442                 elif prefix == 'all':
1443                         self._download_n_results(query, self._max_google_results)
1444                         return
1445                 else:
1446                         try:
1447                                 n = long(prefix)
1448                                 if n <= 0:
1449                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1450                                         return
1451                                 elif n > self._max_google_results:
1452                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1453                                         n = self._max_google_results
1454                                 self._download_n_results(query, n)
1455                                 return
1456                         except ValueError: # parsing prefix as integer fails
1457                                 self._download_n_results(query, 1)
1458                                 return
1459
1460         def _download_n_results(self, query, n):
1461                 """Downloads a specified number of results for a query"""
1462
1463                 video_ids = []
1464                 pagenum = 0
1465
1466                 while True:
1467                         self.report_download_page(query, pagenum)
1468                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10)
1469                         request = urllib2.Request(result_url)
1470                         try:
1471                                 page = urllib2.urlopen(request).read()
1472                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1473                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1474                                 return
1475
1476                         # Extract video identifiers
1477                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1478                                 video_id = mobj.group(1)
1479                                 if video_id not in video_ids:
1480                                         video_ids.append(video_id)
1481                                         if len(video_ids) == n:
1482                                                 # Specified n videos reached
1483                                                 for id in video_ids:
1484                                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1485                                                 return
1486
1487                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1488                                 for id in video_ids:
1489                                         self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1490                                 return
1491
1492                         pagenum = pagenum + 1
1493
1494
1495 class YahooSearchIE(InfoExtractor):
1496         """Information Extractor for Yahoo! Video search queries."""
1497         _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
1498         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1499         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1500         _MORE_PAGES_INDICATOR = r'\s*Next'
1501         _yahoo_ie = None
1502         _max_yahoo_results = 1000
1503         IE_NAME = u'video.yahoo:search'
1504
1505         def __init__(self, yahoo_ie, downloader=None):
1506                 InfoExtractor.__init__(self, downloader)
1507                 self._yahoo_ie = yahoo_ie
1508
1509         def report_download_page(self, query, pagenum):
1510                 """Report attempt to download playlist page with given number."""
1511                 query = query.decode(preferredencoding())
1512                 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1513
1514         def _real_initialize(self):
1515                 self._yahoo_ie.initialize()
1516
1517         def _real_extract(self, query):
1518                 mobj = re.match(self._VALID_URL, query)
1519                 if mobj is None:
1520                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1521                         return
1522
1523                 prefix, query = query.split(':')
1524                 prefix = prefix[8:]
1525                 query = query.encode('utf-8')
1526                 if prefix == '':
1527                         self._download_n_results(query, 1)
1528                         return
1529                 elif prefix == 'all':
1530                         self._download_n_results(query, self._max_yahoo_results)
1531                         return
1532                 else:
1533                         try:
1534                                 n = long(prefix)
1535                                 if n <= 0:
1536                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1537                                         return
1538                                 elif n > self._max_yahoo_results:
1539                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1540                                         n = self._max_yahoo_results
1541                                 self._download_n_results(query, n)
1542                                 return
1543                         except ValueError: # parsing prefix as integer fails
1544                                 self._download_n_results(query, 1)
1545                                 return
1546
1547         def _download_n_results(self, query, n):
1548                 """Downloads a specified number of results for a query"""
1549
1550                 video_ids = []
1551                 already_seen = set()
1552                 pagenum = 1
1553
1554                 while True:
1555                         self.report_download_page(query, pagenum)
1556                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1557                         request = urllib2.Request(result_url)
1558                         try:
1559                                 page = urllib2.urlopen(request).read()
1560                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1561                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1562                                 return
1563
1564                         # Extract video identifiers
1565                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1566                                 video_id = mobj.group(1)
1567                                 if video_id not in already_seen:
1568                                         video_ids.append(video_id)
1569                                         already_seen.add(video_id)
1570                                         if len(video_ids) == n:
1571                                                 # Specified n videos reached
1572                                                 for id in video_ids:
1573                                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1574                                                 return
1575
1576                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1577                                 for id in video_ids:
1578                                         self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1579                                 return
1580
1581                         pagenum = pagenum + 1
1582
1583
1584 class YoutubePlaylistIE(InfoExtractor):
1585         """Information Extractor for YouTube playlists."""
1586
1587         _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
1588         _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
1589         _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;list=PL%s&'
1590         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1591         _youtube_ie = None
1592         IE_NAME = u'youtube:playlist'
1593
1594         def __init__(self, youtube_ie, downloader=None):
1595                 InfoExtractor.__init__(self, downloader)
1596                 self._youtube_ie = youtube_ie
1597
1598         def report_download_page(self, playlist_id, pagenum):
1599                 """Report attempt to download playlist page with given number."""
1600                 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1601
1602         def _real_initialize(self):
1603                 self._youtube_ie.initialize()
1604
1605         def _real_extract(self, url):
1606                 # Extract playlist id
1607                 mobj = re.match(self._VALID_URL, url)
1608                 if mobj is None:
1609                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1610                         return
1611
1612                 # Single video case
1613                 if mobj.group(3) is not None:
1614                         self._youtube_ie.extract(mobj.group(3))
1615                         return
1616
1617                 # Download playlist pages
1618                 # prefix is 'p' as default for playlists but there are other types that need extra care
1619                 playlist_prefix = mobj.group(1)
1620                 if playlist_prefix == 'a':
1621                         playlist_access = 'artist'
1622                 else:
1623                         playlist_prefix = 'p'
1624                         playlist_access = 'view_play_list'
1625                 playlist_id = mobj.group(2)
1626                 video_ids = []
1627                 pagenum = 1
1628
1629                 while True:
1630                         self.report_download_page(playlist_id, pagenum)
1631                         url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
1632                         request = urllib2.Request(url)
1633                         try:
1634                                 page = urllib2.urlopen(request).read()
1635                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1636                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1637                                 return
1638
1639                         # Extract video identifiers
1640                         ids_in_page = []
1641                         for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
1642                                 if mobj.group(1) not in ids_in_page:
1643                                         ids_in_page.append(mobj.group(1))
1644                         video_ids.extend(ids_in_page)
1645
1646                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1647                                 break
1648                         pagenum = pagenum + 1
1649
1650                 playliststart = self._downloader.params.get('playliststart', 1) - 1
1651                 playlistend = self._downloader.params.get('playlistend', -1)
1652                 if playlistend == -1:
1653                         video_ids = video_ids[playliststart:]
1654                 else:
1655                         video_ids = video_ids[playliststart:playlistend]
1656
1657                 for id in video_ids:
1658                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1659                 return
1660
1661
1662 class YoutubeUserIE(InfoExtractor):
1663         """Information Extractor for YouTube users."""
1664
1665         _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
1666         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1667         _GDATA_PAGE_SIZE = 50
1668         _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
1669         _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
1670         _youtube_ie = None
1671         IE_NAME = u'youtube:user'
1672
1673         def __init__(self, youtube_ie, downloader=None):
1674                 InfoExtractor.__init__(self, downloader)
1675                 self._youtube_ie = youtube_ie
1676
1677         def report_download_page(self, username, start_index):
1678                 """Report attempt to download user page."""
1679                 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
1680                                 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
1681
1682         def _real_initialize(self):
1683                 self._youtube_ie.initialize()
1684
1685         def _real_extract(self, url):
1686                 # Extract username
1687                 mobj = re.match(self._VALID_URL, url)
1688                 if mobj is None:
1689                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1690                         return
1691
1692                 username = mobj.group(1)
1693
1694                 # Download video ids using YouTube Data API. Result size per
1695                 # query is limited (currently to 50 videos) so we need to query
1696                 # page by page until there are no video ids - it means we got
1697                 # all of them.
1698
1699                 video_ids = []
1700                 pagenum = 0
1701
1702                 while True:
1703                         start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1704                         self.report_download_page(username, start_index)
1705
1706                         request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
1707
1708                         try:
1709                                 page = urllib2.urlopen(request).read()
1710                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1711                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1712                                 return
1713
1714                         # Extract video identifiers
1715                         ids_in_page = []
1716
1717                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1718                                 if mobj.group(1) not in ids_in_page:
1719                                         ids_in_page.append(mobj.group(1))
1720
1721                         video_ids.extend(ids_in_page)
1722
1723                         # A little optimization - if current page is not
1724                         # "full", ie. does not contain PAGE_SIZE video ids then
1725                         # we can assume that this page is the last one - there
1726                         # are no more ids on further pages - no need to query
1727                         # again.
1728
1729                         if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1730                                 break
1731
1732                         pagenum += 1
1733
1734                 all_ids_count = len(video_ids)
1735                 playliststart = self._downloader.params.get('playliststart', 1) - 1
1736                 playlistend = self._downloader.params.get('playlistend', -1)
1737
1738                 if playlistend == -1:
1739                         video_ids = video_ids[playliststart:]
1740                 else:
1741                         video_ids = video_ids[playliststart:playlistend]
1742
1743                 self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
1744                                 (username, all_ids_count, len(video_ids)))
1745
1746                 for video_id in video_ids:
1747                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id)
1748
1749
1750 class DepositFilesIE(InfoExtractor):
1751         """Information extractor for depositfiles.com"""
1752
1753         _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
1754         IE_NAME = u'DepositFiles'
1755
1756         def __init__(self, downloader=None):
1757                 InfoExtractor.__init__(self, downloader)
1758
1759         def report_download_webpage(self, file_id):
1760                 """Report webpage download."""
1761                 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
1762
1763         def report_extraction(self, file_id):
1764                 """Report information extraction."""
1765                 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
1766
1767         def _real_extract(self, url):
1768                 # At this point we have a new file
1769                 self._downloader.increment_downloads()
1770
1771                 file_id = url.split('/')[-1]
1772                 # Rebuild url in english locale
1773                 url = 'http://depositfiles.com/en/files/' + file_id
1774
1775                 # Retrieve file webpage with 'Free download' button pressed
1776                 free_download_indication = { 'gateway_result' : '1' }
1777                 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
1778                 try:
1779                         self.report_download_webpage(file_id)
1780                         webpage = urllib2.urlopen(request).read()
1781                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1782                         self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
1783                         return
1784
1785                 # Search for the real file URL
1786                 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
1787                 if (mobj is None) or (mobj.group(1) is None):
1788                         # Try to figure out reason of the error.
1789                         mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
1790                         if (mobj is not None) and (mobj.group(1) is not None):
1791                                 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
1792                                 self._downloader.trouble(u'ERROR: %s' % restriction_message)
1793                         else:
1794                                 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
1795                         return
1796
1797                 file_url = mobj.group(1)
1798                 file_extension = os.path.splitext(file_url)[1][1:]
1799
1800                 # Search for file title
1801                 mobj = re.search(r'<b title="(.*?)">', webpage)
1802                 if mobj is None:
1803                         self._downloader.trouble(u'ERROR: unable to extract title')
1804                         return
1805                 file_title = mobj.group(1).decode('utf-8')
1806
1807                 try:
1808                         # Process file information
1809                         self._downloader.process_info({
1810                                 'id':           file_id.decode('utf-8'),
1811                                 'url':          file_url.decode('utf-8'),
1812                                 'uploader':     u'NA',
1813                                 'upload_date':  u'NA',
1814                                 'title':        file_title,
1815                                 'stitle':       file_title,
1816                                 'ext':          file_extension.decode('utf-8'),
1817                                 'format':       u'NA',
1818                                 'player_url':   None,
1819                         })
1820                 except UnavailableVideoError, err:
1821                         self._downloader.trouble(u'ERROR: unable to download file')
1822
1823
1824 class FacebookIE(InfoExtractor):
1825         """Information Extractor for Facebook"""
1826
1827         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
1828         _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
1829         _NETRC_MACHINE = 'facebook'
1830         _available_formats = ['video', 'highqual', 'lowqual']
1831         _video_extensions = {
1832                 'video': 'mp4',
1833                 'highqual': 'mp4',
1834                 'lowqual': 'mp4',
1835         }
1836         IE_NAME = u'facebook'
1837
1838         def __init__(self, downloader=None):
1839                 InfoExtractor.__init__(self, downloader)
1840
1841         def _reporter(self, message):
1842                 """Add header and report message."""
1843                 self._downloader.to_screen(u'[facebook] %s' % message)
1844
1845         def report_login(self):
1846                 """Report attempt to log in."""
1847                 self._reporter(u'Logging in')
1848
1849         def report_video_webpage_download(self, video_id):
1850                 """Report attempt to download video webpage."""
1851                 self._reporter(u'%s: Downloading video webpage' % video_id)
1852
1853         def report_information_extraction(self, video_id):
1854                 """Report attempt to extract video information."""
1855                 self._reporter(u'%s: Extracting video information' % video_id)
1856
1857         def _parse_page(self, video_webpage):
1858                 """Extract video information from page"""
1859                 # General data
1860                 data = {'title': r'\("video_title", "(.*?)"\)',
1861                         'description': r'<div class="datawrap">(.*?)</div>',
1862                         'owner': r'\("video_owner_name", "(.*?)"\)',
1863                         'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
1864                         }
1865                 video_info = {}
1866                 for piece in data.keys():
1867                         mobj = re.search(data[piece], video_webpage)
1868                         if mobj is not None:
1869                                 video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
1870
1871                 # Video urls
1872                 video_urls = {}
1873                 for fmt in self._available_formats:
1874                         mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
1875                         if mobj is not None:
1876                                 # URL is in a Javascript segment inside an escaped Unicode format within
1877                                 # the generally utf-8 page
1878                                 video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
1879                 video_info['video_urls'] = video_urls
1880
1881                 return video_info
1882
1883         def _real_initialize(self):
1884                 if self._downloader is None:
1885                         return
1886
1887                 useremail = None
1888                 password = None
1889                 downloader_params = self._downloader.params
1890
1891                 # Attempt to use provided username and password or .netrc data
1892                 if downloader_params.get('username', None) is not None:
1893                         useremail = downloader_params['username']
1894                         password = downloader_params['password']
1895                 elif downloader_params.get('usenetrc', False):
1896                         try:
1897                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
1898                                 if info is not None:
1899                                         useremail = info[0]
1900                                         password = info[2]
1901                                 else:
1902                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
1903                         except (IOError, netrc.NetrcParseError), err:
1904                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
1905                                 return
1906
1907                 if useremail is None:
1908                         return
1909
1910                 # Log in
1911                 login_form = {
1912                         'email': useremail,
1913                         'pass': password,
1914                         'login': 'Log+In'
1915                         }
1916                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
1917                 try:
1918                         self.report_login()
1919                         login_results = urllib2.urlopen(request).read()
1920                         if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
1921                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
1922                                 return
1923                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1924                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
1925                         return
1926
1927         def _real_extract(self, url):
1928                 mobj = re.match(self._VALID_URL, url)
1929                 if mobj is None:
1930                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1931                         return
1932                 video_id = mobj.group('ID')
1933
1934                 # Get video webpage
1935                 self.report_video_webpage_download(video_id)
1936                 request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
1937                 try:
1938                         page = urllib2.urlopen(request)
1939                         video_webpage = page.read()
1940                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1941                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
1942                         return
1943
1944                 # Start extracting information
1945                 self.report_information_extraction(video_id)
1946
1947                 # Extract information
1948                 video_info = self._parse_page(video_webpage)
1949
1950                 # uploader
1951                 if 'owner' not in video_info:
1952                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1953                         return
1954                 video_uploader = video_info['owner']
1955
1956                 # title
1957                 if 'title' not in video_info:
1958                         self._downloader.trouble(u'ERROR: unable to extract video title')
1959                         return
1960                 video_title = video_info['title']
1961                 video_title = video_title.decode('utf-8')
1962                 video_title = sanitize_title(video_title)
1963
1964                 simple_title = simplify_title(video_title)
1965
1966                 # thumbnail image
1967                 if 'thumbnail' not in video_info:
1968                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
1969                         video_thumbnail = ''
1970                 else:
1971                         video_thumbnail = video_info['thumbnail']
1972
1973                 # upload date
1974                 upload_date = u'NA'
1975                 if 'upload_date' in video_info:
1976                         upload_time = video_info['upload_date']
1977                         timetuple = email.utils.parsedate_tz(upload_time)
1978                         if timetuple is not None:
1979                                 try:
1980                                         upload_date = time.strftime('%Y%m%d', timetuple[0:9])
1981                                 except:
1982                                         pass
1983
1984                 # description
1985                 video_description = video_info.get('description', 'No description available.')
1986
1987                 url_map = video_info['video_urls']
1988                 if len(url_map.keys()) > 0:
1989                         # Decide which formats to download
1990                         req_format = self._downloader.params.get('format', None)
1991                         format_limit = self._downloader.params.get('format_limit', None)
1992
1993                         if format_limit is not None and format_limit in self._available_formats:
1994                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
1995                         else:
1996                                 format_list = self._available_formats
1997                         existing_formats = [x for x in format_list if x in url_map]
1998                         if len(existing_formats) == 0:
1999                                 self._downloader.trouble(u'ERROR: no known formats available for video')
2000                                 return
2001                         if req_format is None:
2002                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2003                         elif req_format == 'worst':
2004                                 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
2005                         elif req_format == '-1':
2006                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2007                         else:
2008                                 # Specific format
2009                                 if req_format not in url_map:
2010                                         self._downloader.trouble(u'ERROR: requested format not available')
2011                                         return
2012                                 video_url_list = [(req_format, url_map[req_format])] # Specific format
2013
2014                 for format_param, video_real_url in video_url_list:
2015
2016                         # At this point we have a new video
2017                         self._downloader.increment_downloads()
2018
2019                         # Extension
2020                         video_extension = self._video_extensions.get(format_param, 'mp4')
2021
2022                         try:
2023                                 # Process video information
2024                                 self._downloader.process_info({
2025                                         'id':           video_id.decode('utf-8'),
2026                                         'url':          video_real_url.decode('utf-8'),
2027                                         'uploader':     video_uploader.decode('utf-8'),
2028                                         'upload_date':  upload_date,
2029                                         'title':        video_title,
2030                                         'stitle':       simple_title,
2031                                         'ext':          video_extension.decode('utf-8'),
2032                                         'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
2033                                         'thumbnail':    video_thumbnail.decode('utf-8'),
2034                                         'description':  video_description.decode('utf-8'),
2035                                         'player_url':   None,
2036                                 })
2037                         except UnavailableVideoError, err:
2038                                 self._downloader.trouble(u'\nERROR: unable to download video')
2039
2040 class BlipTVIE(InfoExtractor):
2041         """Information extractor for blip.tv"""
2042
2043         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2044         _URL_EXT = r'^.*\.([a-z0-9]+)$'
2045         IE_NAME = u'blip.tv'
2046
2047         def report_extraction(self, file_id):
2048                 """Report information extraction."""
2049                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2050
2051         def report_direct_download(self, title):
2052                 """Report information extraction."""
2053                 self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
2054
2055         def _real_extract(self, url):
2056                 mobj = re.match(self._VALID_URL, url)
2057                 if mobj is None:
2058                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2059                         return
2060
2061                 if '?' in url:
2062                         cchar = '&'
2063                 else:
2064                         cchar = '?'
2065                 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2066                 request = urllib2.Request(json_url)
2067                 self.report_extraction(mobj.group(1))
2068                 info = None
2069                 try:
2070                         urlh = urllib2.urlopen(request)
2071                         if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
2072                                 basename = url.split('/')[-1]
2073                                 title,ext = os.path.splitext(basename)
2074                                 title = title.decode('UTF-8')
2075                                 ext = ext.replace('.', '')
2076                                 self.report_direct_download(title)
2077                                 info = {
2078                                         'id': title,
2079                                         'url': url,
2080                                         'title': title,
2081                                         'stitle': simplify_title(title),
2082                                         'ext': ext,
2083                                         'urlhandle': urlh
2084                                 }
2085                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2086                         self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
2087                         return
2088                 if info is None: # Regular URL
2089                         try:
2090                                 json_code = urlh.read()
2091                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2092                                 self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err))
2093                                 return
2094
2095                         try:
2096                                 json_data = json.loads(json_code)
2097                                 if 'Post' in json_data:
2098                                         data = json_data['Post']
2099                                 else:
2100                                         data = json_data
2101
2102                                 upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2103                                 video_url = data['media']['url']
2104                                 umobj = re.match(self._URL_EXT, video_url)
2105                                 if umobj is None:
2106                                         raise ValueError('Can not determine filename extension')
2107                                 ext = umobj.group(1)
2108
2109                                 info = {
2110                                         'id': data['item_id'],
2111                                         'url': video_url,
2112                                         'uploader': data['display_name'],
2113                                         'upload_date': upload_date,
2114                                         'title': data['title'],
2115                                         'stitle': simplify_title(data['title']),
2116                                         'ext': ext,
2117                                         'format': data['media']['mimeType'],
2118                                         'thumbnail': data['thumbnailUrl'],
2119                                         'description': data['description'],
2120                                         'player_url': data['embedUrl']
2121                                 }
2122                         except (ValueError,KeyError), err:
2123                                 self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2124                                 return
2125
2126                 self._downloader.increment_downloads()
2127
2128                 try:
2129                         self._downloader.process_info(info)
2130                 except UnavailableVideoError, err:
2131                         self._downloader.trouble(u'\nERROR: unable to download video')
2132
2133
2134 class MyVideoIE(InfoExtractor):
2135         """Information Extractor for myvideo.de."""
2136
2137         _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2138         IE_NAME = u'myvideo'
2139
2140         def __init__(self, downloader=None):
2141                 InfoExtractor.__init__(self, downloader)
2142
2143         def report_download_webpage(self, video_id):
2144                 """Report webpage download."""
2145                 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2146
2147         def report_extraction(self, video_id):
2148                 """Report information extraction."""
2149                 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2150
2151         def _real_extract(self,url):
2152                 mobj = re.match(self._VALID_URL, url)
2153                 if mobj is None:
2154                         self._download.trouble(u'ERROR: invalid URL: %s' % url)
2155                         return
2156
2157                 video_id = mobj.group(1)
2158
2159                 # Get video webpage
2160                 request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
2161                 try:
2162                         self.report_download_webpage(video_id)
2163                         webpage = urllib2.urlopen(request).read()
2164                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2165                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
2166                         return
2167
2168                 self.report_extraction(video_id)
2169                 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
2170                                  webpage)
2171                 if mobj is None:
2172                         self._downloader.trouble(u'ERROR: unable to extract media URL')
2173                         return
2174                 video_url = mobj.group(1) + ('/%s.flv' % video_id)
2175
2176                 mobj = re.search('<title>([^<]+)</title>', webpage)
2177                 if mobj is None:
2178                         self._downloader.trouble(u'ERROR: unable to extract title')
2179                         return
2180
2181                 video_title = mobj.group(1)
2182                 video_title = sanitize_title(video_title)
2183
2184                 simple_title = simplify_title(video_title)
2185
2186                 try:
2187                         self._downloader.process_info({
2188                                 'id':           video_id,
2189                                 'url':          video_url,
2190                                 'uploader':     u'NA',
2191                                 'upload_date':  u'NA',
2192                                 'title':        video_title,
2193                                 'stitle':       simple_title,
2194                                 'ext':          u'flv',
2195                                 'format':       u'NA',
2196                                 'player_url':   None,
2197                         })
2198                 except UnavailableVideoError:
2199                         self._downloader.trouble(u'\nERROR: Unable to download video')
2200
2201 class ComedyCentralIE(InfoExtractor):
2202         """Information extractor for The Daily Show and Colbert Report """
2203
2204         _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
2205         IE_NAME = u'comedycentral'
2206
2207         def report_extraction(self, episode_id):
2208                 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
2209
2210         def report_config_download(self, episode_id):
2211                 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
2212
2213         def report_index_download(self, episode_id):
2214                 self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
2215
2216         def report_player_url(self, episode_id):
2217                 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
2218
2219         def _real_extract(self, url):
2220                 mobj = re.match(self._VALID_URL, url)
2221                 if mobj is None:
2222                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2223                         return
2224
2225                 if mobj.group('shortname'):
2226                         if mobj.group('shortname') in ('tds', 'thedailyshow'):
2227                                 url = u'http://www.thedailyshow.com/full-episodes/'
2228                         else:
2229                                 url = u'http://www.colbertnation.com/full-episodes/'
2230                         mobj = re.match(self._VALID_URL, url)
2231                         assert mobj is not None
2232
2233                 dlNewest = not mobj.group('episode')
2234                 if dlNewest:
2235                         epTitle = mobj.group('showname')
2236                 else:
2237                         epTitle = mobj.group('episode')
2238
2239                 req = urllib2.Request(url)
2240                 self.report_extraction(epTitle)
2241                 try:
2242                         htmlHandle = urllib2.urlopen(req)
2243                         html = htmlHandle.read()
2244                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2245                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
2246                         return
2247                 if dlNewest:
2248                         url = htmlHandle.geturl()
2249                         mobj = re.match(self._VALID_URL, url)
2250                         if mobj is None:
2251                                 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
2252                                 return
2253                         if mobj.group('episode') == '':
2254                                 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
2255                                 return
2256                         epTitle = mobj.group('episode')
2257
2258                 mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
2259                 if len(mMovieParams) == 0:
2260                         self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
2261                         return
2262
2263                 playerUrl_raw = mMovieParams[0][0]
2264                 self.report_player_url(epTitle)
2265                 try:
2266                         urlHandle = urllib2.urlopen(playerUrl_raw)
2267                         playerUrl = urlHandle.geturl()
2268                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2269                         self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
2270                         return
2271
2272                 uri = mMovieParams[0][1]
2273                 indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
2274                 self.report_index_download(epTitle)
2275                 try:
2276                         indexXml = urllib2.urlopen(indexUrl).read()
2277                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2278                         self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
2279                         return
2280
2281                 idoc = xml.etree.ElementTree.fromstring(indexXml)
2282                 itemEls = idoc.findall('.//item')
2283                 for itemEl in itemEls:
2284                         mediaId = itemEl.findall('./guid')[0].text
2285                         shortMediaId = mediaId.split(':')[-1]
2286                         showId = mediaId.split(':')[-2].replace('.com', '')
2287                         officialTitle = itemEl.findall('./title')[0].text
2288                         officialDate = itemEl.findall('./pubDate')[0].text
2289
2290                         configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
2291                                                 urllib.urlencode({'uri': mediaId}))
2292                         configReq = urllib2.Request(configUrl)
2293                         self.report_config_download(epTitle)
2294                         try:
2295                                 configXml = urllib2.urlopen(configReq).read()
2296                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2297                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
2298                                 return
2299
2300                         cdoc = xml.etree.ElementTree.fromstring(configXml)
2301                         turls = []
2302                         for rendition in cdoc.findall('.//rendition'):
2303                                 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
2304                                 turls.append(finfo)
2305
2306                         if len(turls) == 0:
2307                                 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
2308                                 continue
2309
2310                         # For now, just pick the highest bitrate
2311                         format,video_url = turls[-1]
2312
2313                         self._downloader.increment_downloads()
2314
2315                         effTitle = showId + u'-' + epTitle
2316                         info = {
2317                                 'id': shortMediaId,
2318                                 'url': video_url,
2319                                 'uploader': showId,
2320                                 'upload_date': officialDate,
2321                                 'title': effTitle,
2322                                 'stitle': simplify_title(effTitle),
2323                                 'ext': 'mp4',
2324                                 'format': format,
2325                                 'thumbnail': None,
2326                                 'description': officialTitle,
2327                                 'player_url': playerUrl
2328                         }
2329
2330                         try:
2331                                 self._downloader.process_info(info)
2332                         except UnavailableVideoError, err:
2333                                 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId)
2334                                 continue
2335
2336
2337 class EscapistIE(InfoExtractor):
2338         """Information extractor for The Escapist """
2339
2340         _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
2341         IE_NAME = u'escapist'
2342
2343         def report_extraction(self, showName):
2344                 self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
2345
2346         def report_config_download(self, showName):
2347                 self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
2348
2349         def _real_extract(self, url):
2350                 htmlParser = HTMLParser.HTMLParser()
2351
2352                 mobj = re.match(self._VALID_URL, url)
2353                 if mobj is None:
2354                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2355                         return
2356                 showName = mobj.group('showname')
2357                 videoId = mobj.group('episode')
2358
2359                 self.report_extraction(showName)
2360                 try:
2361                         webPage = urllib2.urlopen(url).read()
2362                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2363                         self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
2364                         return
2365
2366                 descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
2367                 description = htmlParser.unescape(descMatch.group(1))
2368                 imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
2369                 imgUrl = htmlParser.unescape(imgMatch.group(1))
2370                 playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
2371                 playerUrl = htmlParser.unescape(playerUrlMatch.group(1))
2372                 configUrlMatch = re.search('config=(.*)$', playerUrl)
2373                 configUrl = urllib2.unquote(configUrlMatch.group(1))
2374
2375                 self.report_config_download(showName)
2376                 try:
2377                         configJSON = urllib2.urlopen(configUrl).read()
2378                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2379                         self._downloader.trouble(u'ERROR: unable to download configuration: ' + unicode(err))
2380                         return
2381
2382                 # Technically, it's JavaScript, not JSON
2383                 configJSON = configJSON.replace("'", '"')
2384
2385                 try:
2386                         config = json.loads(configJSON)
2387                 except (ValueError,), err:
2388                         self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + unicode(err))
2389                         return
2390
2391                 playlist = config['playlist']
2392                 videoUrl = playlist[1]['url']
2393
2394                 self._downloader.increment_downloads()
2395                 info = {
2396                         'id': videoId,
2397                         'url': videoUrl,
2398                         'uploader': showName,
2399                         'upload_date': None,
2400                         'title': showName,
2401                         'stitle': simplify_title(showName),
2402                         'ext': 'flv',
2403                         'format': 'flv',
2404                         'thumbnail': imgUrl,
2405                         'description': description,
2406                         'player_url': playerUrl,
2407                 }
2408
2409                 try:
2410                         self._downloader.process_info(info)
2411                 except UnavailableVideoError, err:
2412                         self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
2413
2414
2415 class CollegeHumorIE(InfoExtractor):
2416         """Information extractor for collegehumor.com"""
2417
2418         _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
2419         IE_NAME = u'collegehumor'
2420
2421         def report_webpage(self, video_id):
2422                 """Report information extraction."""
2423                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2424
2425         def report_extraction(self, video_id):
2426                 """Report information extraction."""
2427                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2428
2429         def _real_extract(self, url):
2430                 htmlParser = HTMLParser.HTMLParser()
2431
2432                 mobj = re.match(self._VALID_URL, url)
2433                 if mobj is None:
2434                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2435                         return
2436                 video_id = mobj.group('videoid')
2437
2438                 self.report_webpage(video_id)
2439                 request = urllib2.Request(url)
2440                 try:
2441                         webpage = urllib2.urlopen(request).read()
2442                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2443                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2444                         return
2445
2446                 m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
2447                 if m is None:
2448                         self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
2449                         return
2450                 internal_video_id = m.group('internalvideoid')
2451
2452                 info = {
2453                         'id': video_id,
2454                         'internal_id': internal_video_id,
2455                 }
2456
2457                 self.report_extraction(video_id)
2458                 xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
2459                 try:
2460                         metaXml = urllib2.urlopen(xmlUrl).read()
2461                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2462                         self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
2463                         return
2464
2465                 mdoc = xml.etree.ElementTree.fromstring(metaXml)
2466                 try:
2467                         videoNode = mdoc.findall('./video')[0]
2468                         info['description'] = videoNode.findall('./description')[0].text
2469                         info['title'] = videoNode.findall('./caption')[0].text
2470                         info['stitle'] = simplify_title(info['title'])
2471                         info['url'] = videoNode.findall('./file')[0].text
2472                         info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
2473                         info['ext'] = info['url'].rpartition('.')[2]
2474                         info['format'] = info['ext']
2475                 except IndexError:
2476                         self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2477                         return
2478
2479                 self._downloader.increment_downloads()
2480
2481                 try:
2482                         self._downloader.process_info(info)
2483                 except UnavailableVideoError, err:
2484                         self._downloader.trouble(u'\nERROR: unable to download video')
2485
2486
2487 class XVideosIE(InfoExtractor):
2488         """Information extractor for xvideos.com"""
2489
2490         _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
2491         IE_NAME = u'xvideos'
2492
2493         def report_webpage(self, video_id):
2494                 """Report information extraction."""
2495                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2496
2497         def report_extraction(self, video_id):
2498                 """Report information extraction."""
2499                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2500
2501         def _real_extract(self, url):
2502                 htmlParser = HTMLParser.HTMLParser()
2503
2504                 mobj = re.match(self._VALID_URL, url)
2505                 if mobj is None:
2506                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2507                         return
2508                 video_id = mobj.group(1).decode('utf-8')
2509
2510                 self.report_webpage(video_id)
2511
2512                 request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
2513                 try:
2514                         webpage = urllib2.urlopen(request).read()
2515                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2516                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2517                         return
2518
2519                 self.report_extraction(video_id)
2520
2521
2522                 # Extract video URL
2523                 mobj = re.search(r'flv_url=(.+?)&', webpage)
2524                 if mobj is None:
2525                         self._downloader.trouble(u'ERROR: unable to extract video url')
2526                         return
2527                 video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
2528
2529
2530                 # Extract title
2531                 mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
2532                 if mobj is None:
2533                         self._downloader.trouble(u'ERROR: unable to extract video title')
2534                         return
2535                 video_title = mobj.group(1).decode('utf-8')
2536
2537
2538                 # Extract video thumbnail
2539                 mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
2540                 if mobj is None:
2541                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
2542                         return
2543                 video_thumbnail = mobj.group(1).decode('utf-8')
2544
2545
2546
2547                 self._downloader.increment_downloads()
2548                 info = {
2549                         'id': video_id,
2550                         'url': video_url,
2551                         'uploader': None,
2552                         'upload_date': None,
2553                         'title': video_title,
2554                         'stitle': simplify_title(video_title),
2555                         'ext': 'flv',
2556                         'format': 'flv',
2557                         'thumbnail': video_thumbnail,
2558                         'description': None,
2559                         'player_url': None,
2560                 }
2561
2562                 try:
2563                         self._downloader.process_info(info)
2564                 except UnavailableVideoError, err:
2565                         self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
2566
2567
2568 class SoundcloudIE(InfoExtractor):
2569         """Information extractor for soundcloud.com
2570            To access the media, the uid of the song and a stream token
2571            must be extracted from the page source and the script must make
2572            a request to media.soundcloud.com/crossdomain.xml. Then
2573            the media can be grabbed by requesting from an url composed
2574            of the stream token and uid
2575          """
2576
2577         _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
2578         IE_NAME = u'soundcloud'
2579
2580         def __init__(self, downloader=None):
2581                 InfoExtractor.__init__(self, downloader)
2582
2583         def report_webpage(self, video_id):
2584                 """Report information extraction."""
2585                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2586
2587         def report_extraction(self, video_id):
2588                 """Report information extraction."""
2589                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2590
2591         def _real_extract(self, url):
2592                 htmlParser = HTMLParser.HTMLParser()
2593
2594                 mobj = re.match(self._VALID_URL, url)
2595                 if mobj is None:
2596                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2597                         return
2598
2599                 # extract uploader (which is in the url)
2600                 uploader = mobj.group(1).decode('utf-8')
2601                 # extract simple title (uploader + slug of song title)
2602                 slug_title =  mobj.group(2).decode('utf-8')
2603                 simple_title = uploader + '-' + slug_title
2604
2605                 self.report_webpage('%s/%s' % (uploader, slug_title))
2606
2607                 request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
2608                 try:
2609                         webpage = urllib2.urlopen(request).read()
2610                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2611                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2612                         return
2613
2614                 self.report_extraction('%s/%s' % (uploader, slug_title))
2615
2616                 # extract uid and stream token that soundcloud hands out for access
2617                 mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
2618                 if mobj:
2619                         video_id = mobj.group(1)
2620                         stream_token = mobj.group(2)
2621
2622                 # extract unsimplified title
2623                 mobj = re.search('"title":"(.*?)",', webpage)
2624                 if mobj:
2625                         title = mobj.group(1)
2626
2627                 # construct media url (with uid/token)
2628                 mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
2629                 mediaURL = mediaURL % (video_id, stream_token)
2630
2631                 # description
2632                 description = u'No description available'
2633                 mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
2634                 if mobj:
2635                         description = mobj.group(1)
2636
2637                 # upload date
2638                 upload_date = None
2639                 mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
2640                 if mobj:
2641                         try:
2642                                 upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
2643                         except Exception, e:
2644                                 print str(e)
2645
2646                 # for soundcloud, a request to a cross domain is required for cookies
2647                 request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
2648
2649                 try:
2650                         self._downloader.process_info({
2651                                 'id':           video_id.decode('utf-8'),
2652                                 'url':          mediaURL,
2653                                 'uploader':     uploader.decode('utf-8'),
2654                                 'upload_date':  upload_date,
2655                                 'title':        simple_title.decode('utf-8'),
2656                                 'stitle':       simple_title.decode('utf-8'),
2657                                 'ext':          u'mp3',
2658                                 'format':       u'NA',
2659                                 'player_url':   None,
2660                                 'description': description.decode('utf-8')
2661                         })
2662                 except UnavailableVideoError:
2663                         self._downloader.trouble(u'\nERROR: unable to download video')
2664
2665
2666 class InfoQIE(InfoExtractor):
2667         """Information extractor for infoq.com"""
2668
2669         _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
2670         IE_NAME = u'infoq'
2671
2672         def report_webpage(self, video_id):
2673                 """Report information extraction."""
2674                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2675
2676         def report_extraction(self, video_id):
2677                 """Report information extraction."""
2678                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2679
2680         def _real_extract(self, url):
2681                 htmlParser = HTMLParser.HTMLParser()
2682
2683                 mobj = re.match(self._VALID_URL, url)
2684                 if mobj is None:
2685                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2686                         return
2687
2688                 self.report_webpage(url)
2689
2690                 request = urllib2.Request(url)
2691                 try:
2692                         webpage = urllib2.urlopen(request).read()
2693                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2694                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2695                         return
2696
2697                 self.report_extraction(url)
2698
2699
2700                 # Extract video URL
2701                 mobj = re.search(r"jsclassref='([^']*)'", webpage)
2702                 if mobj is None:
2703                         self._downloader.trouble(u'ERROR: unable to extract video url')
2704                         return
2705                 video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
2706
2707
2708                 # Extract title
2709                 mobj = re.search(r'contentTitle = "(.*?)";', webpage)
2710                 if mobj is None:
2711                         self._downloader.trouble(u'ERROR: unable to extract video title')
2712                         return
2713                 video_title = mobj.group(1).decode('utf-8')
2714
2715                 # Extract description
2716                 video_description = u'No description available.'
2717                 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
2718                 if mobj is not None:
2719                         video_description = mobj.group(1).decode('utf-8')
2720
2721                 video_filename = video_url.split('/')[-1]
2722                 video_id, extension = video_filename.split('.')
2723
2724                 self._downloader.increment_downloads()
2725                 info = {
2726                         'id': video_id,
2727                         'url': video_url,
2728                         'uploader': None,
2729                         'upload_date': None,
2730                         'title': video_title,
2731                         'stitle': simplify_title(video_title),
2732                         'ext': extension,
2733                         'format': extension, # Extension is always(?) mp4, but seems to be flv
2734                         'thumbnail': None,
2735                         'description': video_description,
2736                         'player_url': None,
2737                 }
2738
2739                 try:
2740                         self._downloader.process_info(info)
2741                 except UnavailableVideoError, err:
2742                         self._downloader.trouble(u'\nERROR: unable to download ' + video_url)
2743
2744 class MixcloudIE(InfoExtractor):
2745         """Information extractor for www.mixcloud.com"""
2746         _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
2747         IE_NAME = u'mixcloud'
2748
2749         def __init__(self, downloader=None):
2750                 InfoExtractor.__init__(self, downloader)
2751
2752         def report_download_json(self, file_id):
2753                 """Report JSON download."""
2754                 self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
2755
2756         def report_extraction(self, file_id):
2757                 """Report information extraction."""
2758                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2759
2760         def get_urls(self, jsonData, fmt, bitrate='best'):
2761                 """Get urls from 'audio_formats' section in json"""
2762                 file_url = None
2763                 try:
2764                         bitrate_list = jsonData[fmt]
2765                         if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
2766                                 bitrate = max(bitrate_list) # select highest
2767
2768                         url_list = jsonData[fmt][bitrate]
2769                 except TypeError: # we have no bitrate info.
2770                         url_list = jsonData[fmt]
2771
2772                 return url_list
2773
2774         def check_urls(self, url_list):
2775                 """Returns 1st active url from list"""
2776                 for url in url_list:
2777                         try:
2778                                 urllib2.urlopen(url)
2779                                 return url
2780                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2781                                 url = None
2782
2783                 return None
2784
2785         def _print_formats(self, formats):
2786                 print 'Available formats:'
2787                 for fmt in formats.keys():
2788                         for b in formats[fmt]:
2789                                 try:
2790                                         ext = formats[fmt][b][0]
2791                                         print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])
2792                                 except TypeError: # we have no bitrate info
2793                                         ext = formats[fmt][0]
2794                                         print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])
2795                                         break
2796
2797         def _real_extract(self, url):
2798                 mobj = re.match(self._VALID_URL, url)
2799                 if mobj is None:
2800                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2801                         return
2802                 # extract uploader & filename from url
2803                 uploader = mobj.group(1).decode('utf-8')
2804                 file_id = uploader + "-" + mobj.group(2).decode('utf-8')
2805
2806                 # construct API request
2807                 file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
2808                 # retrieve .json file with links to files
2809                 request = urllib2.Request(file_url)
2810                 try:
2811                         self.report_download_json(file_url)
2812                         jsonData = urllib2.urlopen(request).read()
2813                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2814                         self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err))
2815                         return
2816
2817                 # parse JSON
2818                 json_data = json.loads(jsonData)
2819                 player_url = json_data['player_swf_url']
2820                 formats = dict(json_data['audio_formats'])
2821
2822                 req_format = self._downloader.params.get('format', None)
2823                 bitrate = None
2824
2825                 if self._downloader.params.get('listformats', None):
2826                         self._print_formats(formats)
2827                         return
2828
2829                 if req_format is None or req_format == 'best':
2830                         for format_param in formats.keys():
2831                                 url_list = self.get_urls(formats, format_param)
2832                                 # check urls
2833                                 file_url = self.check_urls(url_list)
2834                                 if file_url is not None:
2835                                         break # got it!
2836                 else:
2837                         if req_format not in formats.keys():
2838                                 self._downloader.trouble(u'ERROR: format is not available')
2839                                 return
2840
2841                         url_list = self.get_urls(formats, req_format)
2842                         file_url = self.check_urls(url_list)
2843                         format_param = req_format
2844
2845                 # We have audio
2846                 self._downloader.increment_downloads()
2847                 try:
2848                         # Process file information
2849                         self._downloader.process_info({
2850                                 'id': file_id.decode('utf-8'),
2851                                 'url': file_url.decode('utf-8'),
2852                                 'uploader':     uploader.decode('utf-8'),
2853                                 'upload_date': u'NA',
2854                                 'title': json_data['name'],
2855                                 'stitle': simplify_title(json_data['name']),
2856                                 'ext': file_url.split('.')[-1].decode('utf-8'),
2857                                 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
2858                                 'thumbnail': json_data['thumbnail_url'],
2859                                 'description': json_data['description'],
2860                                 'player_url': player_url.decode('utf-8'),
2861                         })
2862                 except UnavailableVideoError, err:
2863                         self._downloader.trouble(u'ERROR: unable to download file')
2864
2865 class StanfordOpenClassroomIE(InfoExtractor):
2866         """Information extractor for Stanford's Open ClassRoom"""
2867
2868         _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
2869         IE_NAME = u'stanfordoc'
2870
2871         def report_download_webpage(self, objid):
2872                 """Report information extraction."""
2873                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
2874
2875         def report_extraction(self, video_id):
2876                 """Report information extraction."""
2877                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2878
2879         def _real_extract(self, url):
2880                 mobj = re.match(self._VALID_URL, url)
2881                 if mobj is None:
2882                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2883                         return
2884
2885                 if mobj.group('course') and mobj.group('video'): # A specific video
2886                         course = mobj.group('course')
2887                         video = mobj.group('video')
2888                         info = {
2889                                 'id': simplify_title(course + '_' + video),
2890                         }
2891
2892                         self.report_extraction(info['id'])
2893                         baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
2894                         xmlUrl = baseUrl + video + '.xml'
2895                         try:
2896                                 metaXml = urllib2.urlopen(xmlUrl).read()
2897                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2898                                 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
2899                                 return
2900                         mdoc = xml.etree.ElementTree.fromstring(metaXml)
2901                         try:
2902                                 info['title'] = mdoc.findall('./title')[0].text
2903                                 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
2904                         except IndexError:
2905                                 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2906                                 return
2907                         info['stitle'] = simplify_title(info['title'])
2908                         info['ext'] = info['url'].rpartition('.')[2]
2909                         info['format'] = info['ext']
2910                         self._downloader.increment_downloads()
2911                         try:
2912                                 self._downloader.process_info(info)
2913                         except UnavailableVideoError, err:
2914                                 self._downloader.trouble(u'\nERROR: unable to download video')
2915                 elif mobj.group('course'): # A course page
2916                         unescapeHTML = HTMLParser.HTMLParser().unescape
2917
2918                         course = mobj.group('course')
2919                         info = {
2920                                 'id': simplify_title(course),
2921                                 'type': 'playlist',
2922                         }
2923
2924                         self.report_download_webpage(info['id'])
2925                         try:
2926                                 coursepage = urllib2.urlopen(url).read()
2927                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2928                                 self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
2929                                 return
2930
2931                         m = re.search('<h1>([^<]+)</h1>', coursepage)
2932                         if m:
2933                                 info['title'] = unescapeHTML(m.group(1))
2934                         else:
2935                                 info['title'] = info['id']
2936                         info['stitle'] = simplify_title(info['title'])
2937
2938                         m = re.search('<description>([^<]+)</description>', coursepage)
2939                         if m:
2940                                 info['description'] = unescapeHTML(m.group(1))
2941
2942                         links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
2943                         info['list'] = [
2944                                 {
2945                                         'type': 'reference',
2946                                         'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
2947                                 }
2948                                         for vpage in links]
2949
2950                         for entry in info['list']:
2951                                 assert entry['type'] == 'reference'
2952                                 self.extract(entry['url'])
2953                 else: # Root page
2954                         unescapeHTML = HTMLParser.HTMLParser().unescape
2955
2956                         info = {
2957                                 'id': 'Stanford OpenClassroom',
2958                                 'type': 'playlist',
2959                         }
2960
2961                         self.report_download_webpage(info['id'])
2962                         rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
2963                         try:
2964                                 rootpage = urllib2.urlopen(rootURL).read()
2965                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2966                                 self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
2967                                 return
2968
2969                         info['title'] = info['id']
2970                         info['stitle'] = simplify_title(info['title'])
2971
2972                         links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
2973                         info['list'] = [
2974                                 {
2975                                         'type': 'reference',
2976                                         'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
2977                                 }
2978                                         for cpage in links]
2979
2980                         for entry in info['list']:
2981                                 assert entry['type'] == 'reference'
2982                                 self.extract(entry['url'])
2983
2984 class MTVIE(InfoExtractor):
2985         """Information extractor for MTV.com"""
2986
2987         _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
2988         IE_NAME = u'mtv'
2989
2990         def report_webpage(self, video_id):
2991                 """Report information extraction."""
2992                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2993
2994         def report_extraction(self, video_id):
2995                 """Report information extraction."""
2996                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2997
2998         def _real_extract(self, url):
2999                 mobj = re.match(self._VALID_URL, url)
3000                 if mobj is None:
3001                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3002                         return
3003                 if not mobj.group('proto'):
3004                         url = 'http://' + url
3005                 video_id = mobj.group('videoid')
3006                 self.report_webpage(video_id)
3007
3008                 request = urllib2.Request(url)
3009                 try:
3010                         webpage = urllib2.urlopen(request).read()
3011                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3012                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
3013                         return
3014
3015                 mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
3016                 if mobj is None:
3017                         self._downloader.trouble(u'ERROR: unable to extract song name')
3018                         return
3019                 song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3020                 mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
3021                 if mobj is None:
3022                         self._downloader.trouble(u'ERROR: unable to extract performer')
3023                         return
3024                 performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3025                 video_title = performer + ' - ' + song_name
3026
3027                 mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
3028                 if mobj is None:
3029                         self._downloader.trouble(u'ERROR: unable to mtvn_uri')
3030                         return
3031                 mtvn_uri = mobj.group(1)
3032
3033                 mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
3034                 if mobj is None:
3035                         self._downloader.trouble(u'ERROR: unable to extract content id')
3036                         return
3037                 content_id = mobj.group(1)
3038
3039                 videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
3040                 self.report_extraction(video_id)
3041                 request = urllib2.Request(videogen_url)
3042                 try:
3043                         metadataXml = urllib2.urlopen(request).read()
3044                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3045                         self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err))
3046                         return
3047
3048                 mdoc = xml.etree.ElementTree.fromstring(metadataXml)
3049                 renditions = mdoc.findall('.//rendition')
3050
3051                 # For now, always pick the highest quality.
3052                 rendition = renditions[-1]
3053
3054                 try:
3055                         _,_,ext = rendition.attrib['type'].partition('/')
3056                         format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
3057                         video_url = rendition.find('./src').text
3058                 except KeyError:
3059                         self._downloader.trouble('Invalid rendition field.')
3060                         return
3061
3062                 self._downloader.increment_downloads()
3063                 info = {
3064                         'id': video_id,
3065                         'url': video_url,
3066                         'uploader': performer,
3067                         'title': video_title,
3068                         'stitle': simplify_title(video_title),
3069                         'ext': ext,
3070                         'format': format,
3071                 }
3072
3073                 try:
3074                         self._downloader.process_info(info)
3075                 except UnavailableVideoError, err:
3076                         self._downloader.trouble(u'\nERROR: unable to download ' + video_id)