_ Git - youtube-dl/blob - youtube_dl/InfoExtractors.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import datetime
   5 import HTMLParser
   6 import httplib
   7 import netrc
   8 import os
   9 import re
  10 import socket
  11 import time
  12 import urllib
  13 import urllib2
  14 import email.utils
  15 import xml.etree.ElementTree
  16 import random
  17 import math
  18 from urlparse import parse_qs, urlparse
  19
  20 try:
  21         import cStringIO as StringIO
  22 except ImportError:
  23         import StringIO
  24
  25 from utils import *
  26
  27
  28 class InfoExtractor(object):
  29         """Information Extractor class.
  30
  31         Information extractors are the classes that, given a URL, extract
  32         information from the video (or videos) the URL refers to. This
  33         information includes the real video URL, the video title and simplified
  34         title, author and others. The information is stored in a dictionary
  35         which is then passed to the FileDownloader. The FileDownloader
  36         processes this information possibly downloading the video to the file
  37         system, among other possible outcomes. The dictionaries must include
  38         the following fields:
  39
  40         id:             Video identifier.
  41         url:            Final video URL.
  42         uploader:       Nickname of the video uploader.
  43         title:          Literal title.
  44         ext:            Video filename extension.
  45         format:         Video format.
  46         player_url:     SWF Player URL (may be None).
  47
  48         The following fields are optional. Their primary purpose is to allow
  49         youtube-dl to serve as the backend for a video search function, such
  50         as the one in youtube2mp3.  They are only used when their respective
  51         forced printing functions are called:
  52
  53         thumbnail:      Full URL to a video thumbnail image.
  54         description:    One-line video description.
  55
  56         Subclasses of this one should re-define the _real_initialize() and
  57         _real_extract() methods and define a _VALID_URL regexp.
  58         Probably, they should also be added to the list of extractors.
  59         """
  60
  61         _ready = False
  62         _downloader = None
  63
  64         def __init__(self, downloader=None):
  65                 """Constructor. Receives an optional downloader."""
  66                 self._ready = False
  67                 self.set_downloader(downloader)
  68
  69         def suitable(self, url):
  70                 """Receives a URL and returns True if suitable for this IE."""
  71                 return re.match(self._VALID_URL, url) is not None
  72
  73         def initialize(self):
  74                 """Initializes an instance (authentication, etc)."""
  75                 if not self._ready:
  76                         self._real_initialize()
  77                         self._ready = True
  78
  79         def extract(self, url):
  80                 """Extracts URL information and returns it in list of dicts."""
  81                 self.initialize()
  82                 return self._real_extract(url)
  83
  84         def set_downloader(self, downloader):
  85                 """Sets the downloader for this IE."""
  86                 self._downloader = downloader
  87
  88         def _real_initialize(self):
  89                 """Real initialization process. Redefine in subclasses."""
  90                 pass
  91
  92         def _real_extract(self, url):
  93                 """Real extraction process. Redefine in subclasses."""
  94                 pass
  95
  96
  97
  98
  99 class YoutubeIE(InfoExtractor):
 100         """Information extractor for youtube.com."""
 101
 102         _VALID_URL = r"""^
 103                          (
 104                              (?:https?://)?                                       # http(s):// (optional)
 105                              (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
 106                                 tube\.majestyc\.net/)                             # the various hostnames, with wildcard subdomains
 107                              (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 108                              (?!view_play_list|my_playlists|artist|playlist)      # ignore playlist URLs
 109                              (?:                                                  # the various things that can precede the ID:
 110                                  (?:(?:v|embed|e)/)                               # v/ or embed/ or e/
 111                                  |(?:                                             # or the v= param in all its forms
 112                                      (?:watch(?:_popup)?(?:\.php)?)?              # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 113                                      (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 114                                      (?:.+&)?                                     # any other preceding param (like /?s=tuff&v=xxxx)
 115                                      v=
 116                                  )
 117                              )?                                                   # optional -> youtube.com/xxxx is OK
 118                          )?                                                       # all until now is optional -> you can pass the naked ID
 119                          ([0-9A-Za-z_-]+)                                         # here is it! the YouTube video ID
 120                          (?(1).+)?                                                # if we found the ID, everything can follow
 121                          $"""
 122         _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 123         _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
 124         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 125         _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 126         _NETRC_MACHINE = 'youtube'
 127         # Listed in order of quality
 128         _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
 129         _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
 130         _video_extensions = {
 131                 '13': '3gp',
 132                 '17': 'mp4',
 133                 '18': 'mp4',
 134                 '22': 'mp4',
 135                 '37': 'mp4',
 136                 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 137                 '43': 'webm',
 138                 '44': 'webm',
 139                 '45': 'webm',
 140                 '46': 'webm',
 141         }
 142         _video_dimensions = {
 143                 '5': '240x400',
 144                 '6': '???',
 145                 '13': '???',
 146                 '17': '144x176',
 147                 '18': '360x640',
 148                 '22': '720x1280',
 149                 '34': '360x640',
 150                 '35': '480x854',
 151                 '37': '1080x1920',
 152                 '38': '3072x4096',
 153                 '43': '360x640',
 154                 '44': '480x854',
 155                 '45': '720x1280',
 156                 '46': '1080x1920',
 157         }
 158         IE_NAME = u'youtube'
 159
 160         def suitable(self, url):
 161                 """Receives a URL and returns True if suitable for this IE."""
 162                 return re.match(self._VALID_URL, url, re.VERBOSE) is not None
 163
 164         def report_lang(self):
 165                 """Report attempt to set language."""
 166                 self._downloader.to_screen(u'[youtube] Setting language')
 167
 168         def report_login(self):
 169                 """Report attempt to log in."""
 170                 self._downloader.to_screen(u'[youtube] Logging in')
 171
 172         def report_age_confirmation(self):
 173                 """Report attempt to confirm age."""
 174                 self._downloader.to_screen(u'[youtube] Confirming age')
 175
 176         def report_video_webpage_download(self, video_id):
 177                 """Report attempt to download video webpage."""
 178                 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
 179
 180         def report_video_info_webpage_download(self, video_id):
 181                 """Report attempt to download video info webpage."""
 182                 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
 183
 184         def report_video_subtitles_download(self, video_id):
 185                 """Report attempt to download video info webpage."""
 186                 self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
 187
 188         def report_information_extraction(self, video_id):
 189                 """Report attempt to extract video information."""
 190                 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
 191
 192         def report_unavailable_format(self, video_id, format):
 193                 """Report extracted video URL."""
 194                 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
 195
 196         def report_rtmp_download(self):
 197                 """Indicate the download will use the RTMP protocol."""
 198                 self._downloader.to_screen(u'[youtube] RTMP download detected')
 199
 200         def _closed_captions_xml_to_srt(self, xml_string):
 201                 srt = ''
 202                 texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
 203                 # TODO parse xml instead of regex
 204                 for n, (start, dur_tag, dur, caption) in enumerate(texts):
 205                         if not dur: dur = '4'
 206                         start = float(start)
 207                         end = start + float(dur)
 208                         start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
 209                         end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
 210                         caption = unescapeHTML(caption)
 211                         caption = unescapeHTML(caption) # double cycle, intentional
 212                         srt += str(n+1) + '\n'
 213                         srt += start + ' --> ' + end + '\n'
 214                         srt += caption + '\n\n'
 215                 return srt
 216
 217         def _print_formats(self, formats):
 218                 print 'Available formats:'
 219                 for x in formats:
 220                         print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))
 221
 222         def _real_initialize(self):
 223                 if self._downloader is None:
 224                         return
 225
 226                 username = None
 227                 password = None
 228                 downloader_params = self._downloader.params
 229
 230                 # Attempt to use provided username and password or .netrc data
 231                 if downloader_params.get('username', None) is not None:
 232                         username = downloader_params['username']
 233                         password = downloader_params['password']
 234                 elif downloader_params.get('usenetrc', False):
 235                         try:
 236                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 237                                 if info is not None:
 238                                         username = info[0]
 239                                         password = info[2]
 240                                 else:
 241                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 242                         except (IOError, netrc.NetrcParseError), err:
 243                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 244                                 return
 245
 246                 # Set language
 247                 request = urllib2.Request(self._LANG_URL)
 248                 try:
 249                         self.report_lang()
 250                         urllib2.urlopen(request).read()
 251                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 252                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 253                         return
 254
 255                 # No authentication to be performed
 256                 if username is None:
 257                         return
 258
 259                 # Log in
 260                 login_form = {
 261                                 'current_form': 'loginForm',
 262                                 'next':         '/',
 263                                 'action_login': 'Log In',
 264                                 'username':     username,
 265                                 'password':     password,
 266                                 }
 267                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
 268                 try:
 269                         self.report_login()
 270                         login_results = urllib2.urlopen(request).read()
 271                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 272                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 273                                 return
 274                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 275                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 276                         return
 277
 278                 # Confirm age
 279                 age_form = {
 280                                 'next_url':             '/',
 281                                 'action_confirm':       'Confirm',
 282                                 }
 283                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
 284                 try:
 285                         self.report_age_confirmation()
 286                         age_results = urllib2.urlopen(request).read()
 287                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 288                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 289                         return
 290
 291         def _real_extract(self, url):
 292                 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
 293                 mobj = re.search(self._NEXT_URL_RE, url)
 294                 if mobj:
 295                         url = 'http://www.youtube.com/' + urllib.unquote(mobj.group(1)).lstrip('/')
 296
 297                 # Extract video id from URL
 298                 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
 299                 if mobj is None:
 300                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 301                         return
 302                 video_id = mobj.group(2)
 303
 304                 # Get video webpage
 305                 self.report_video_webpage_download(video_id)
 306                 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
 307                 try:
 308                         video_webpage = urllib2.urlopen(request).read()
 309                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 310                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 311                         return
 312
 313                 # Attempt to extract SWF player URL
 314                 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
 315                 if mobj is not None:
 316                         player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
 317                 else:
 318                         player_url = None
 319
 320                 # Get video info
 321                 self.report_video_info_webpage_download(video_id)
 322                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
 323                         video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
 324                                         % (video_id, el_type))
 325                         request = urllib2.Request(video_info_url)
 326                         try:
 327                                 video_info_webpage = urllib2.urlopen(request).read()
 328                                 video_info = parse_qs(video_info_webpage)
 329                                 if 'token' in video_info:
 330                                         break
 331                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 332                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 333                                 return
 334                 if 'token' not in video_info:
 335                         if 'reason' in video_info:
 336                                 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
 337                         else:
 338                                 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
 339                         return
 340
 341                 # Check for "rental" videos
 342                 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
 343                         self._downloader.trouble(u'ERROR: "rental" videos not supported')
 344                         return
 345
 346                 # Start extracting information
 347                 self.report_information_extraction(video_id)
 348
 349                 # uploader
 350                 if 'author' not in video_info:
 351                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 352                         return
 353                 video_uploader = urllib.unquote_plus(video_info['author'][0])
 354
 355                 # title
 356                 if 'title' not in video_info:
 357                         self._downloader.trouble(u'ERROR: unable to extract video title')
 358                         return
 359                 video_title = urllib.unquote_plus(video_info['title'][0])
 360                 video_title = video_title.decode('utf-8')
 361
 362                 # thumbnail image
 363                 if 'thumbnail_url' not in video_info:
 364                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
 365                         video_thumbnail = ''
 366                 else:   # don't panic if we can't find it
 367                         video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
 368
 369                 # upload date
 370                 upload_date = u'NA'
 371                 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
 372                 if mobj is not None:
 373                         upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
 374                         format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
 375                         for expression in format_expressions:
 376                                 try:
 377                                         upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
 378                                 except:
 379                                         pass
 380
 381                 # description
 382                 video_description = get_element_by_id("eow-description", video_webpage.decode('utf8'))
 383                 if video_description: video_description = clean_html(video_description)
 384                 else: video_description = ''
 385
 386                 # closed captions
 387                 video_subtitles = None
 388                 if self._downloader.params.get('writesubtitles', False):
 389                         try:
 390                                 self.report_video_subtitles_download(video_id)
 391                                 request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
 392                                 try:
 393                                         srt_list = urllib2.urlopen(request).read()
 394                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 395                                         raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
 396                                 srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
 397                                 srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
 398                                 if not srt_lang_list:
 399                                         raise Trouble(u'WARNING: video has no closed captions')
 400                                 if self._downloader.params.get('subtitleslang', False):
 401                                         srt_lang = self._downloader.params.get('subtitleslang')
 402                                 elif 'en' in srt_lang_list:
 403                                         srt_lang = 'en'
 404                                 else:
 405                                         srt_lang = srt_lang_list.keys()[0]
 406                                 if not srt_lang in srt_lang_list:
 407                                         raise Trouble(u'WARNING: no closed captions found in the specified language')
 408                                 request = urllib2.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
 409                                 try:
 410                                         srt_xml = urllib2.urlopen(request).read()
 411                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 412                                         raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
 413                                 if not srt_xml:
 414                                         raise Trouble(u'WARNING: unable to download video subtitles')
 415                                 video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
 416                         except Trouble as trouble:
 417                                 self._downloader.trouble(trouble[0])
 418
 419                 if 'length_seconds' not in video_info:
 420                         self._downloader.trouble(u'WARNING: unable to extract video duration')
 421                         video_duration = ''
 422                 else:
 423                         video_duration = urllib.unquote_plus(video_info['length_seconds'][0])
 424
 425                 # token
 426                 video_token = urllib.unquote_plus(video_info['token'][0])
 427
 428                 # Decide which formats to download
 429                 req_format = self._downloader.params.get('format', None)
 430
 431                 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 432                         self.report_rtmp_download()
 433                         video_url_list = [(None, video_info['conn'][0])]
 434                 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
 435                         url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
 436                         url_data = [parse_qs(uds) for uds in url_data_strs]
 437                         url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
 438                         url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
 439
 440                         format_limit = self._downloader.params.get('format_limit', None)
 441                         available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
 442                         if format_limit is not None and format_limit in available_formats:
 443                                 format_list = available_formats[available_formats.index(format_limit):]
 444                         else:
 445                                 format_list = available_formats
 446                         existing_formats = [x for x in format_list if x in url_map]
 447                         if len(existing_formats) == 0:
 448                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 449                                 return
 450                         if self._downloader.params.get('listformats', None):
 451                                 self._print_formats(existing_formats)
 452                                 return
 453                         if req_format is None or req_format == 'best':
 454                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
 455                         elif req_format == 'worst':
 456                                 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
 457                         elif req_format in ('-1', 'all'):
 458                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
 459                         else:
 460                                 # Specific formats. We pick the first in a slash-delimeted sequence.
 461                                 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
 462                                 req_formats = req_format.split('/')
 463                                 video_url_list = None
 464                                 for rf in req_formats:
 465                                         if rf in url_map:
 466                                                 video_url_list = [(rf, url_map[rf])]
 467                                                 break
 468                                 if video_url_list is None:
 469                                         self._downloader.trouble(u'ERROR: requested format not available')
 470                                         return
 471                 else:
 472                         self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
 473                         return
 474
 475                 results = []
 476                 for format_param, video_real_url in video_url_list:
 477                         # Extension
 478                         video_extension = self._video_extensions.get(format_param, 'flv')
 479
 480                         results.append({
 481                                 'id':           video_id.decode('utf-8'),
 482                                 'url':          video_real_url.decode('utf-8'),
 483                                 'uploader':     video_uploader.decode('utf-8'),
 484                                 'upload_date':  upload_date,
 485                                 'title':        video_title,
 486                                 'ext':          video_extension.decode('utf-8'),
 487                                 'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
 488                                 'thumbnail':    video_thumbnail.decode('utf-8'),
 489                                 'description':  video_description,
 490                                 'player_url':   player_url,
 491                                 'subtitles':    video_subtitles,
 492                                 'duration':             video_duration
 493                         })
 494                 return results
 495
 496
 497 class MetacafeIE(InfoExtractor):
 498         """Information Extractor for metacafe.com."""
 499
 500         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 501         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 502         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 503         IE_NAME = u'metacafe'
 504
 505         def __init__(self, downloader=None):
 506                 InfoExtractor.__init__(self, downloader)
 507
 508         def report_disclaimer(self):
 509                 """Report disclaimer retrieval."""
 510                 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
 511
 512         def report_age_confirmation(self):
 513                 """Report attempt to confirm age."""
 514                 self._downloader.to_screen(u'[metacafe] Confirming age')
 515
 516         def report_download_webpage(self, video_id):
 517                 """Report webpage download."""
 518                 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
 519
 520         def report_extraction(self, video_id):
 521                 """Report information extraction."""
 522                 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
 523
 524         def _real_initialize(self):
 525                 # Retrieve disclaimer
 526                 request = urllib2.Request(self._DISCLAIMER)
 527                 try:
 528                         self.report_disclaimer()
 529                         disclaimer = urllib2.urlopen(request).read()
 530                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 531                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 532                         return
 533
 534                 # Confirm age
 535                 disclaimer_form = {
 536                         'filters': '0',
 537                         'submit': "Continue - I'm over 18",
 538                         }
 539                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
 540                 try:
 541                         self.report_age_confirmation()
 542                         disclaimer = urllib2.urlopen(request).read()
 543                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 544                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 545                         return
 546
 547         def _real_extract(self, url):
 548                 # Extract id and simplified title from URL
 549                 mobj = re.match(self._VALID_URL, url)
 550                 if mobj is None:
 551                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 552                         return
 553
 554                 video_id = mobj.group(1)
 555
 556                 # Check if video comes from YouTube
 557                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 558                 if mobj2 is not None:
 559                         self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
 560                         return
 561
 562                 # Retrieve video webpage to extract further information
 563                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 564                 try:
 565                         self.report_download_webpage(video_id)
 566                         webpage = urllib2.urlopen(request).read()
 567                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 568                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 569                         return
 570
 571                 # Extract URL, uploader and title from webpage
 572                 self.report_extraction(video_id)
 573                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 574                 if mobj is not None:
 575                         mediaURL = urllib.unquote(mobj.group(1))
 576                         video_extension = mediaURL[-3:]
 577
 578                         # Extract gdaKey if available
 579                         mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 580                         if mobj is None:
 581                                 video_url = mediaURL
 582                         else:
 583                                 gdaKey = mobj.group(1)
 584                                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 585                 else:
 586                         mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
 587                         if mobj is None:
 588                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 589                                 return
 590                         vardict = parse_qs(mobj.group(1))
 591                         if 'mediaData' not in vardict:
 592                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 593                                 return
 594                         mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
 595                         if mobj is None:
 596                                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 597                                 return
 598                         mediaURL = mobj.group(1).replace('\\/', '/')
 599                         video_extension = mediaURL[-3:]
 600                         video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
 601
 602                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 603                 if mobj is None:
 604                         self._downloader.trouble(u'ERROR: unable to extract title')
 605                         return
 606                 video_title = mobj.group(1).decode('utf-8')
 607
 608                 mobj = re.search(r'submitter=(.*?);', webpage)
 609                 if mobj is None:
 610                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 611                         return
 612                 video_uploader = mobj.group(1)
 613
 614                 return [{
 615                         'id':           video_id.decode('utf-8'),
 616                         'url':          video_url.decode('utf-8'),
 617                         'uploader':     video_uploader.decode('utf-8'),
 618                         'upload_date':  u'NA',
 619                         'title':        video_title,
 620                         'ext':          video_extension.decode('utf-8'),
 621                         'format':       u'NA',
 622                         'player_url':   None,
 623                 }]
 624
 625
 626 class DailymotionIE(InfoExtractor):
 627         """Information Extractor for Dailymotion"""
 628
 629         _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
 630         IE_NAME = u'dailymotion'
 631
 632         def __init__(self, downloader=None):
 633                 InfoExtractor.__init__(self, downloader)
 634
 635         def report_download_webpage(self, video_id):
 636                 """Report webpage download."""
 637                 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
 638
 639         def report_extraction(self, video_id):
 640                 """Report information extraction."""
 641                 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
 642
 643         def _real_extract(self, url):
 644                 # Extract id and simplified title from URL
 645                 mobj = re.match(self._VALID_URL, url)
 646                 if mobj is None:
 647                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 648                         return
 649
 650                 video_id = mobj.group(1).split('_')[0].split('?')[0]
 651
 652                 video_extension = 'mp4'
 653
 654                 # Retrieve video webpage to extract further information
 655                 request = urllib2.Request(url)
 656                 request.add_header('Cookie', 'family_filter=off')
 657                 try:
 658                         self.report_download_webpage(video_id)
 659                         webpage = urllib2.urlopen(request).read()
 660                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 661                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 662                         return
 663
 664                 # Extract URL, uploader and title from webpage
 665                 self.report_extraction(video_id)
 666                 mobj = re.search(r'\s*var flashvars = (.*)', webpage)
 667                 if mobj is None:
 668                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 669                         return
 670                 flashvars = urllib.unquote(mobj.group(1))
 671
 672                 for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
 673                         if key in flashvars:
 674                                 max_quality = key
 675                                 self._downloader.to_screen(u'[dailymotion] Using %s' % key)
 676                                 break
 677                 else:
 678                         self._downloader.trouble(u'ERROR: unable to extract video URL')
 679                         return
 680
 681                 mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
 682                 if mobj is None:
 683                         self._downloader.trouble(u'ERROR: unable to extract video URL')
 684                         return
 685
 686                 video_url = urllib.unquote(mobj.group(1)).replace('\\/', '/')
 687
 688                 # TODO: support choosing qualities
 689
 690                 mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
 691                 if mobj is None:
 692                         self._downloader.trouble(u'ERROR: unable to extract title')
 693                         return
 694                 video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
 695
 696                 video_uploader = u'NA'
 697                 mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
 698                 if mobj is None:
 699                         # lookin for official user
 700                         mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
 701                         if mobj_official is None:
 702                                 self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
 703                         else:
 704                                 video_uploader = mobj_official.group(1)
 705                 else:
 706                         video_uploader = mobj.group(1)
 707
 708                 video_upload_date = u'NA'
 709                 mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
 710                 if mobj is not None:
 711                         video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
 712
 713                 return [{
 714                         'id':           video_id.decode('utf-8'),
 715                         'url':          video_url.decode('utf-8'),
 716                         'uploader':     video_uploader.decode('utf-8'),
 717                         'upload_date':  video_upload_date,
 718                         'title':        video_title,
 719                         'ext':          video_extension.decode('utf-8'),
 720                         'format':       u'NA',
 721                         'player_url':   None,
 722                 }]
 723
 724
 725 class GoogleIE(InfoExtractor):
 726         """Information extractor for video.google.com."""
 727
 728         _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
 729         IE_NAME = u'video.google'
 730
 731         def __init__(self, downloader=None):
 732                 InfoExtractor.__init__(self, downloader)
 733
 734         def report_download_webpage(self, video_id):
 735                 """Report webpage download."""
 736                 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
 737
 738         def report_extraction(self, video_id):
 739                 """Report information extraction."""
 740                 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
 741
 742         def _real_extract(self, url):
 743                 # Extract id from URL
 744                 mobj = re.match(self._VALID_URL, url)
 745                 if mobj is None:
 746                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 747                         return
 748
 749                 video_id = mobj.group(1)
 750
 751                 video_extension = 'mp4'
 752
 753                 # Retrieve video webpage to extract further information
 754                 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
 755                 try:
 756                         self.report_download_webpage(video_id)
 757                         webpage = urllib2.urlopen(request).read()
 758                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 759                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 760                         return
 761
 762                 # Extract URL, uploader, and title from webpage
 763                 self.report_extraction(video_id)
 764                 mobj = re.search(r"download_url:'([^']+)'", webpage)
 765                 if mobj is None:
 766                         video_extension = 'flv'
 767                         mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
 768                 if mobj is None:
 769                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 770                         return
 771                 mediaURL = urllib.unquote(mobj.group(1))
 772                 mediaURL = mediaURL.replace('\\x3d', '\x3d')
 773                 mediaURL = mediaURL.replace('\\x26', '\x26')
 774
 775                 video_url = mediaURL
 776
 777                 mobj = re.search(r'<title>(.*)</title>', webpage)
 778                 if mobj is None:
 779                         self._downloader.trouble(u'ERROR: unable to extract title')
 780                         return
 781                 video_title = mobj.group(1).decode('utf-8')
 782
 783                 # Extract video description
 784                 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
 785                 if mobj is None:
 786                         self._downloader.trouble(u'ERROR: unable to extract video description')
 787                         return
 788                 video_description = mobj.group(1).decode('utf-8')
 789                 if not video_description:
 790                         video_description = 'No description available.'
 791
 792                 # Extract video thumbnail
 793                 if self._downloader.params.get('forcethumbnail', False):
 794                         request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
 795                         try:
 796                                 webpage = urllib2.urlopen(request).read()
 797                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 798                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 799                                 return
 800                         mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
 801                         if mobj is None:
 802                                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
 803                                 return
 804                         video_thumbnail = mobj.group(1)
 805                 else:   # we need something to pass to process_info
 806                         video_thumbnail = ''
 807
 808                 return [{
 809                         'id':           video_id.decode('utf-8'),
 810                         'url':          video_url.decode('utf-8'),
 811                         'uploader':     u'NA',
 812                         'upload_date':  u'NA',
 813                         'title':        video_title,
 814                         'ext':          video_extension.decode('utf-8'),
 815                         'format':       u'NA',
 816                         'player_url':   None,
 817                 }]
 818
 819
 820 class PhotobucketIE(InfoExtractor):
 821         """Information extractor for photobucket.com."""
 822
 823         _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
 824         IE_NAME = u'photobucket'
 825
 826         def __init__(self, downloader=None):
 827                 InfoExtractor.__init__(self, downloader)
 828
 829         def report_download_webpage(self, video_id):
 830                 """Report webpage download."""
 831                 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
 832
 833         def report_extraction(self, video_id):
 834                 """Report information extraction."""
 835                 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
 836
 837         def _real_extract(self, url):
 838                 # Extract id from URL
 839                 mobj = re.match(self._VALID_URL, url)
 840                 if mobj is None:
 841                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 842                         return
 843
 844                 video_id = mobj.group(1)
 845
 846                 video_extension = 'flv'
 847
 848                 # Retrieve video webpage to extract further information
 849                 request = urllib2.Request(url)
 850                 try:
 851                         self.report_download_webpage(video_id)
 852                         webpage = urllib2.urlopen(request).read()
 853                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 854                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 855                         return
 856
 857                 # Extract URL, uploader, and title from webpage
 858                 self.report_extraction(video_id)
 859                 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
 860                 if mobj is None:
 861                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 862                         return
 863                 mediaURL = urllib.unquote(mobj.group(1))
 864
 865                 video_url = mediaURL
 866
 867                 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
 868                 if mobj is None:
 869                         self._downloader.trouble(u'ERROR: unable to extract title')
 870                         return
 871                 video_title = mobj.group(1).decode('utf-8')
 872
 873                 video_uploader = mobj.group(2).decode('utf-8')
 874
 875                 return [{
 876                         'id':           video_id.decode('utf-8'),
 877                         'url':          video_url.decode('utf-8'),
 878                         'uploader':     video_uploader,
 879                         'upload_date':  u'NA',
 880                         'title':        video_title,
 881                         'ext':          video_extension.decode('utf-8'),
 882                         'format':       u'NA',
 883                         'player_url':   None,
 884                 }]
 885
 886
 887 class YahooIE(InfoExtractor):
 888         """Information extractor for video.yahoo.com."""
 889
 890         # _VALID_URL matches all Yahoo! Video URLs
 891         # _VPAGE_URL matches only the extractable '/watch/' URLs
 892         _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
 893         _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
 894         IE_NAME = u'video.yahoo'
 895
 896         def __init__(self, downloader=None):
 897                 InfoExtractor.__init__(self, downloader)
 898
 899         def report_download_webpage(self, video_id):
 900                 """Report webpage download."""
 901                 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
 902
 903         def report_extraction(self, video_id):
 904                 """Report information extraction."""
 905                 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
 906
 907         def _real_extract(self, url, new_video=True):
 908                 # Extract ID from URL
 909                 mobj = re.match(self._VALID_URL, url)
 910                 if mobj is None:
 911                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 912                         return
 913
 914                 video_id = mobj.group(2)
 915                 video_extension = 'flv'
 916
 917                 # Rewrite valid but non-extractable URLs as
 918                 # extractable English language /watch/ URLs
 919                 if re.match(self._VPAGE_URL, url) is None:
 920                         request = urllib2.Request(url)
 921                         try:
 922                                 webpage = urllib2.urlopen(request).read()
 923                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 924                                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 925                                 return
 926
 927                         mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
 928                         if mobj is None:
 929                                 self._downloader.trouble(u'ERROR: Unable to extract id field')
 930                                 return
 931                         yahoo_id = mobj.group(1)
 932
 933                         mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
 934                         if mobj is None:
 935                                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
 936                                 return
 937                         yahoo_vid = mobj.group(1)
 938
 939                         url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
 940                         return self._real_extract(url, new_video=False)
 941
 942                 # Retrieve video webpage to extract further information
 943                 request = urllib2.Request(url)
 944                 try:
 945                         self.report_download_webpage(video_id)
 946                         webpage = urllib2.urlopen(request).read()
 947                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 948                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
 949                         return
 950
 951                 # Extract uploader and title from webpage
 952                 self.report_extraction(video_id)
 953                 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
 954                 if mobj is None:
 955                         self._downloader.trouble(u'ERROR: unable to extract video title')
 956                         return
 957                 video_title = mobj.group(1).decode('utf-8')
 958
 959                 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
 960                 if mobj is None:
 961                         self._downloader.trouble(u'ERROR: unable to extract video uploader')
 962                         return
 963                 video_uploader = mobj.group(1).decode('utf-8')
 964
 965                 # Extract video thumbnail
 966                 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
 967                 if mobj is None:
 968                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
 969                         return
 970                 video_thumbnail = mobj.group(1).decode('utf-8')
 971
 972                 # Extract video description
 973                 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
 974                 if mobj is None:
 975                         self._downloader.trouble(u'ERROR: unable to extract video description')
 976                         return
 977                 video_description = mobj.group(1).decode('utf-8')
 978                 if not video_description:
 979                         video_description = 'No description available.'
 980
 981                 # Extract video height and width
 982                 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
 983                 if mobj is None:
 984                         self._downloader.trouble(u'ERROR: unable to extract video height')
 985                         return
 986                 yv_video_height = mobj.group(1)
 987
 988                 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
 989                 if mobj is None:
 990                         self._downloader.trouble(u'ERROR: unable to extract video width')
 991                         return
 992                 yv_video_width = mobj.group(1)
 993
 994                 # Retrieve video playlist to extract media URL
 995                 # I'm not completely sure what all these options are, but we
 996                 # seem to need most of them, otherwise the server sends a 401.
 997                 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
 998                 yv_bitrate = '700'  # according to Wikipedia this is hard-coded
 999                 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1000                                 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1001                                 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1002                 try:
1003                         self.report_download_webpage(video_id)
1004                         webpage = urllib2.urlopen(request).read()
1005                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1006                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1007                         return
1008
1009                 # Extract media URL from playlist XML
1010                 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1011                 if mobj is None:
1012                         self._downloader.trouble(u'ERROR: Unable to extract media URL')
1013                         return
1014                 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1015                 video_url = unescapeHTML(video_url)
1016
1017                 return [{
1018                         'id':           video_id.decode('utf-8'),
1019                         'url':          video_url,
1020                         'uploader':     video_uploader,
1021                         'upload_date':  u'NA',
1022                         'title':        video_title,
1023                         'ext':          video_extension.decode('utf-8'),
1024                         'thumbnail':    video_thumbnail.decode('utf-8'),
1025                         'description':  video_description,
1026                         'thumbnail':    video_thumbnail,
1027                         'player_url':   None,
1028                 }]
1029
1030
1031 class VimeoIE(InfoExtractor):
1032         """Information extractor for vimeo.com."""
1033
1034         # _VALID_URL matches Vimeo URLs
1035         _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
1036         IE_NAME = u'vimeo'
1037
1038         def __init__(self, downloader=None):
1039                 InfoExtractor.__init__(self, downloader)
1040
1041         def report_download_webpage(self, video_id):
1042                 """Report webpage download."""
1043                 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1044
1045         def report_extraction(self, video_id):
1046                 """Report information extraction."""
1047                 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1048
1049         def _real_extract(self, url, new_video=True):
1050                 # Extract ID from URL
1051                 mobj = re.match(self._VALID_URL, url)
1052                 if mobj is None:
1053                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1054                         return
1055
1056                 video_id = mobj.group(1)
1057
1058                 # Retrieve video webpage to extract further information
1059                 request = urllib2.Request(url, None, std_headers)
1060                 try:
1061                         self.report_download_webpage(video_id)
1062                         webpage = urllib2.urlopen(request).read()
1063                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1064                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1065                         return
1066
1067                 # Now we begin extracting as much information as we can from what we
1068                 # retrieved. First we extract the information common to all extractors,
1069                 # and latter we extract those that are Vimeo specific.
1070                 self.report_extraction(video_id)
1071
1072                 # Extract the config JSON
1073                 config = webpage.split(' = {config:')[1].split(',assets:')[0]
1074                 try:
1075                         config = json.loads(config)
1076                 except:
1077                         self._downloader.trouble(u'ERROR: unable to extract info section')
1078                         return
1079
1080                 # Extract title
1081                 video_title = config["video"]["title"]
1082
1083                 # Extract uploader
1084                 video_uploader = config["video"]["owner"]["name"]
1085
1086                 # Extract video thumbnail
1087                 video_thumbnail = config["video"]["thumbnail"]
1088
1089                 # Extract video description
1090                 video_description = get_element_by_id("description", webpage.decode('utf8'))
1091                 if video_description: video_description = clean_html(video_description)
1092                 else: video_description = ''
1093
1094                 # Extract upload date
1095                 video_upload_date = u'NA'
1096                 mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
1097                 if mobj is not None:
1098                         video_upload_date = mobj.group(1)
1099
1100                 # Vimeo specific: extract request signature and timestamp
1101                 sig = config['request']['signature']
1102                 timestamp = config['request']['timestamp']
1103
1104                 # Vimeo specific: extract video codec and quality information
1105                 # TODO bind to format param
1106                 codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
1107                 for codec in codecs:
1108                         if codec[0] in config["video"]["files"]:
1109                                 video_codec = codec[0]
1110                                 video_extension = codec[1]
1111                                 if 'hd' in config["video"]["files"][codec[0]]: quality = 'hd'
1112                                 else: quality = 'sd'
1113                                 break
1114                 else:
1115                         self._downloader.trouble(u'ERROR: no known codec found')
1116                         return
1117
1118                 video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
1119                                         %(video_id, sig, timestamp, quality, video_codec.upper())
1120
1121                 return [{
1122                         'id':           video_id,
1123                         'url':          video_url,
1124                         'uploader':     video_uploader,
1125                         'upload_date':  video_upload_date,
1126                         'title':        video_title,
1127                         'ext':          video_extension,
1128                         'thumbnail':    video_thumbnail,
1129                         'description':  video_description,
1130                         'player_url':   None,
1131                 }]
1132
1133
1134 class GenericIE(InfoExtractor):
1135         """Generic last-resort information extractor."""
1136
1137         _VALID_URL = r'.*'
1138         IE_NAME = u'generic'
1139
1140         def __init__(self, downloader=None):
1141                 InfoExtractor.__init__(self, downloader)
1142
1143         def report_download_webpage(self, video_id):
1144                 """Report webpage download."""
1145                 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1146                 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1147
1148         def report_extraction(self, video_id):
1149                 """Report information extraction."""
1150                 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1151
1152         def report_following_redirect(self, new_url):
1153                 """Report information extraction."""
1154                 self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
1155
1156         def _test_redirect(self, url):
1157                 """Check if it is a redirect, like url shorteners, in case restart chain."""
1158                 class HeadRequest(urllib2.Request):
1159                         def get_method(self):
1160                                 return "HEAD"
1161
1162                 class HEADRedirectHandler(urllib2.HTTPRedirectHandler):
1163                         """
1164                         Subclass the HTTPRedirectHandler to make it use our
1165                         HeadRequest also on the redirected URL
1166                         """
1167                         def redirect_request(self, req, fp, code, msg, headers, newurl):
1168                                 if code in (301, 302, 303, 307):
1169                                         newurl = newurl.replace(' ', '%20')
1170                                         newheaders = dict((k,v) for k,v in req.headers.items()
1171                                                                           if k.lower() not in ("content-length", "content-type"))
1172                                         return HeadRequest(newurl,
1173                                                                            headers=newheaders,
1174                                                                            origin_req_host=req.get_origin_req_host(),
1175                                                                            unverifiable=True)
1176                                 else:
1177                                         raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
1178
1179                 class HTTPMethodFallback(urllib2.BaseHandler):
1180                         """
1181                         Fallback to GET if HEAD is not allowed (405 HTTP error)
1182                         """
1183                         def http_error_405(self, req, fp, code, msg, headers):
1184                                 fp.read()
1185                                 fp.close()
1186
1187                                 newheaders = dict((k,v) for k,v in req.headers.items()
1188                                                                   if k.lower() not in ("content-length", "content-type"))
1189                                 return self.parent.open(urllib2.Request(req.get_full_url(),
1190                                                                                                  headers=newheaders,
1191                                                                                                  origin_req_host=req.get_origin_req_host(),
1192                                                                                                  unverifiable=True))
1193
1194                 # Build our opener
1195                 opener = urllib2.OpenerDirector()
1196                 for handler in [urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler,
1197                                                 HTTPMethodFallback, HEADRedirectHandler,
1198                                                 urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler]:
1199                         opener.add_handler(handler())
1200
1201                 response = opener.open(HeadRequest(url))
1202                 new_url = response.geturl()
1203
1204                 if url == new_url: return False
1205
1206                 self.report_following_redirect(new_url)
1207                 self._downloader.download([new_url])
1208                 return True
1209
1210         def _real_extract(self, url):
1211                 if self._test_redirect(url): return
1212
1213                 video_id = url.split('/')[-1]
1214                 request = urllib2.Request(url)
1215                 try:
1216                         self.report_download_webpage(video_id)
1217                         webpage = urllib2.urlopen(request).read()
1218                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1219                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1220                         return
1221                 except ValueError, err:
1222                         # since this is the last-resort InfoExtractor, if
1223                         # this error is thrown, it'll be thrown here
1224                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1225                         return
1226
1227                 self.report_extraction(video_id)
1228                 # Start with something easy: JW Player in SWFObject
1229                 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1230                 if mobj is None:
1231                         # Broaden the search a little bit
1232                         mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1233                 if mobj is None:
1234                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1235                         return
1236
1237                 # It's possible that one of the regexes
1238                 # matched, but returned an empty group:
1239                 if mobj.group(1) is None:
1240                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1241                         return
1242
1243                 video_url = urllib.unquote(mobj.group(1))
1244                 video_id = os.path.basename(video_url)
1245
1246                 # here's a fun little line of code for you:
1247                 video_extension = os.path.splitext(video_id)[1][1:]
1248                 video_id = os.path.splitext(video_id)[0]
1249
1250                 # it's tempting to parse this further, but you would
1251                 # have to take into account all the variations like
1252                 #   Video Title - Site Name
1253                 #   Site Name | Video Title
1254                 #   Video Title - Tagline | Site Name
1255                 # and so on and so forth; it's just not practical
1256                 mobj = re.search(r'<title>(.*)</title>', webpage)
1257                 if mobj is None:
1258                         self._downloader.trouble(u'ERROR: unable to extract title')
1259                         return
1260                 video_title = mobj.group(1).decode('utf-8')
1261
1262                 # video uploader is domain name
1263                 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1264                 if mobj is None:
1265                         self._downloader.trouble(u'ERROR: unable to extract title')
1266                         return
1267                 video_uploader = mobj.group(1).decode('utf-8')
1268
1269                 return [{
1270                         'id':           video_id.decode('utf-8'),
1271                         'url':          video_url.decode('utf-8'),
1272                         'uploader':     video_uploader,
1273                         'upload_date':  u'NA',
1274                         'title':        video_title,
1275                         'ext':          video_extension.decode('utf-8'),
1276                         'format':       u'NA',
1277                         'player_url':   None,
1278                 }]
1279
1280
1281 class YoutubeSearchIE(InfoExtractor):
1282         """Information Extractor for YouTube search queries."""
1283         _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
1284         _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1285         _max_youtube_results = 1000
1286         IE_NAME = u'youtube:search'
1287
1288         def __init__(self, downloader=None):
1289                 InfoExtractor.__init__(self, downloader)
1290
1291         def report_download_page(self, query, pagenum):
1292                 """Report attempt to download search page with given number."""
1293                 query = query.decode(preferredencoding())
1294                 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1295
1296         def _real_extract(self, query):
1297                 mobj = re.match(self._VALID_URL, query)
1298                 if mobj is None:
1299                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1300                         return
1301
1302                 prefix, query = query.split(':')
1303                 prefix = prefix[8:]
1304                 query = query.encode('utf-8')
1305                 if prefix == '':
1306                         self._download_n_results(query, 1)
1307                         return
1308                 elif prefix == 'all':
1309                         self._download_n_results(query, self._max_youtube_results)
1310                         return
1311                 else:
1312                         try:
1313                                 n = long(prefix)
1314                                 if n <= 0:
1315                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1316                                         return
1317                                 elif n > self._max_youtube_results:
1318                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1319                                         n = self._max_youtube_results
1320                                 self._download_n_results(query, n)
1321                                 return
1322                         except ValueError: # parsing prefix as integer fails
1323                                 self._download_n_results(query, 1)
1324                                 return
1325
1326         def _download_n_results(self, query, n):
1327                 """Downloads a specified number of results for a query"""
1328
1329                 video_ids = []
1330                 pagenum = 0
1331                 limit = n
1332
1333                 while (50 * pagenum) < limit:
1334                         self.report_download_page(query, pagenum+1)
1335                         result_url = self._API_URL % (urllib.quote_plus(query), (50*pagenum)+1)
1336                         request = urllib2.Request(result_url)
1337                         try:
1338                                 data = urllib2.urlopen(request).read()
1339                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1340                                 self._downloader.trouble(u'ERROR: unable to download API page: %s' % str(err))
1341                                 return
1342                         api_response = json.loads(data)['data']
1343
1344                         new_ids = list(video['id'] for video in api_response['items'])
1345                         video_ids += new_ids
1346
1347                         limit = min(n, api_response['totalItems'])
1348                         pagenum += 1
1349
1350                 if len(video_ids) > n:
1351                         video_ids = video_ids[:n]
1352                 for id in video_ids:
1353                         self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1354                 return
1355
1356
1357 class GoogleSearchIE(InfoExtractor):
1358         """Information Extractor for Google Video search queries."""
1359         _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
1360         _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1361         _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
1362         _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
1363         _max_google_results = 1000
1364         IE_NAME = u'video.google:search'
1365
1366         def __init__(self, downloader=None):
1367                 InfoExtractor.__init__(self, downloader)
1368
1369         def report_download_page(self, query, pagenum):
1370                 """Report attempt to download playlist page with given number."""
1371                 query = query.decode(preferredencoding())
1372                 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1373
1374         def _real_extract(self, query):
1375                 mobj = re.match(self._VALID_URL, query)
1376                 if mobj is None:
1377                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1378                         return
1379
1380                 prefix, query = query.split(':')
1381                 prefix = prefix[8:]
1382                 query = query.encode('utf-8')
1383                 if prefix == '':
1384                         self._download_n_results(query, 1)
1385                         return
1386                 elif prefix == 'all':
1387                         self._download_n_results(query, self._max_google_results)
1388                         return
1389                 else:
1390                         try:
1391                                 n = long(prefix)
1392                                 if n <= 0:
1393                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1394                                         return
1395                                 elif n > self._max_google_results:
1396                                         self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1397                                         n = self._max_google_results
1398                                 self._download_n_results(query, n)
1399                                 return
1400                         except ValueError: # parsing prefix as integer fails
1401                                 self._download_n_results(query, 1)
1402                                 return
1403
1404         def _download_n_results(self, query, n):
1405                 """Downloads a specified number of results for a query"""
1406
1407                 video_ids = []
1408                 pagenum = 0
1409
1410                 while True:
1411                         self.report_download_page(query, pagenum)
1412                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10)
1413                         request = urllib2.Request(result_url)
1414                         try:
1415                                 page = urllib2.urlopen(request).read()
1416                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1417                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1418                                 return
1419
1420                         # Extract video identifiers
1421                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1422                                 video_id = mobj.group(1)
1423                                 if video_id not in video_ids:
1424                                         video_ids.append(video_id)
1425                                         if len(video_ids) == n:
1426                                                 # Specified n videos reached
1427                                                 for id in video_ids:
1428                                                         self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1429                                                 return
1430
1431                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1432                                 for id in video_ids:
1433                                         self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1434                                 return
1435
1436                         pagenum = pagenum + 1
1437
1438
1439 class YahooSearchIE(InfoExtractor):
1440         """Information Extractor for Yahoo! Video search queries."""
1441         _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
1442         _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1443         _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1444         _MORE_PAGES_INDICATOR = r'\s*Next'
1445         _max_yahoo_results = 1000
1446         IE_NAME = u'video.yahoo:search'
1447
1448         def __init__(self, downloader=None):
1449                 InfoExtractor.__init__(self, downloader)
1450
1451         def report_download_page(self, query, pagenum):
1452                 """Report attempt to download playlist page with given number."""
1453                 query = query.decode(preferredencoding())
1454                 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1455
1456         def _real_extract(self, query):
1457                 mobj = re.match(self._VALID_URL, query)
1458                 if mobj is None:
1459                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1460                         return
1461
1462                 prefix, query = query.split(':')
1463                 prefix = prefix[8:]
1464                 query = query.encode('utf-8')
1465                 if prefix == '':
1466                         self._download_n_results(query, 1)
1467                         return
1468                 elif prefix == 'all':
1469                         self._download_n_results(query, self._max_yahoo_results)
1470                         return
1471                 else:
1472                         try:
1473                                 n = long(prefix)
1474                                 if n <= 0:
1475                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1476                                         return
1477                                 elif n > self._max_yahoo_results:
1478                                         self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1479                                         n = self._max_yahoo_results
1480                                 self._download_n_results(query, n)
1481                                 return
1482                         except ValueError: # parsing prefix as integer fails
1483                                 self._download_n_results(query, 1)
1484                                 return
1485
1486         def _download_n_results(self, query, n):
1487                 """Downloads a specified number of results for a query"""
1488
1489                 video_ids = []
1490                 already_seen = set()
1491                 pagenum = 1
1492
1493                 while True:
1494                         self.report_download_page(query, pagenum)
1495                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1496                         request = urllib2.Request(result_url)
1497                         try:
1498                                 page = urllib2.urlopen(request).read()
1499                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1500                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1501                                 return
1502
1503                         # Extract video identifiers
1504                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1505                                 video_id = mobj.group(1)
1506                                 if video_id not in already_seen:
1507                                         video_ids.append(video_id)
1508                                         already_seen.add(video_id)
1509                                         if len(video_ids) == n:
1510                                                 # Specified n videos reached
1511                                                 for id in video_ids:
1512                                                         self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1513                                                 return
1514
1515                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1516                                 for id in video_ids:
1517                                         self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1518                                 return
1519
1520                         pagenum = pagenum + 1
1521
1522
1523 class YoutubePlaylistIE(InfoExtractor):
1524         """Information Extractor for YouTube playlists."""
1525
1526         _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
1527         _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
1528         _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
1529         _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
1530         IE_NAME = u'youtube:playlist'
1531
1532         def __init__(self, downloader=None):
1533                 InfoExtractor.__init__(self, downloader)
1534
1535         def report_download_page(self, playlist_id, pagenum):
1536                 """Report attempt to download playlist page with given number."""
1537                 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1538
1539         def _real_extract(self, url):
1540                 # Extract playlist id
1541                 mobj = re.match(self._VALID_URL, url)
1542                 if mobj is None:
1543                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1544                         return
1545
1546                 # Single video case
1547                 if mobj.group(3) is not None:
1548                         self._downloader.download([mobj.group(3)])
1549                         return
1550
1551                 # Download playlist pages
1552                 # prefix is 'p' as default for playlists but there are other types that need extra care
1553                 playlist_prefix = mobj.group(1)
1554                 if playlist_prefix == 'a':
1555                         playlist_access = 'artist'
1556                 else:
1557                         playlist_prefix = 'p'
1558                         playlist_access = 'view_play_list'
1559                 playlist_id = mobj.group(2)
1560                 video_ids = []
1561                 pagenum = 1
1562
1563                 while True:
1564                         self.report_download_page(playlist_id, pagenum)
1565                         url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
1566                         request = urllib2.Request(url)
1567                         try:
1568                                 page = urllib2.urlopen(request).read()
1569                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1570                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1571                                 return
1572
1573                         # Extract video identifiers
1574                         ids_in_page = []
1575                         for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
1576                                 if mobj.group(1) not in ids_in_page:
1577                                         ids_in_page.append(mobj.group(1))
1578                         video_ids.extend(ids_in_page)
1579
1580                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1581                                 break
1582                         pagenum = pagenum + 1
1583
1584                 playliststart = self._downloader.params.get('playliststart', 1) - 1
1585                 playlistend = self._downloader.params.get('playlistend', -1)
1586                 if playlistend == -1:
1587                         video_ids = video_ids[playliststart:]
1588                 else:
1589                         video_ids = video_ids[playliststart:playlistend]
1590
1591                 for id in video_ids:
1592                         self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1593                 return
1594
1595
1596 class YoutubeChannelIE(InfoExtractor):
1597         """Information Extractor for YouTube channels."""
1598
1599         _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
1600         _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1601         _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
1602         IE_NAME = u'youtube:channel'
1603
1604         def report_download_page(self, channel_id, pagenum):
1605                 """Report attempt to download channel page with given number."""
1606                 self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
1607
1608         def _real_extract(self, url):
1609                 # Extract channel id
1610                 mobj = re.match(self._VALID_URL, url)
1611                 if mobj is None:
1612                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1613                         return
1614
1615                 # Download channel pages
1616                 channel_id = mobj.group(1)
1617                 video_ids = []
1618                 pagenum = 1
1619
1620                 while True:
1621                         self.report_download_page(channel_id, pagenum)
1622                         url = self._TEMPLATE_URL % (channel_id, pagenum)
1623                         request = urllib2.Request(url)
1624                         try:
1625                                 page = urllib2.urlopen(request).read()
1626                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1627                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1628                                 return
1629
1630                         # Extract video identifiers
1631                         ids_in_page = []
1632                         for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
1633                                 if mobj.group(1) not in ids_in_page:
1634                                         ids_in_page.append(mobj.group(1))
1635                         video_ids.extend(ids_in_page)
1636
1637                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1638                                 break
1639                         pagenum = pagenum + 1
1640
1641                 for id in video_ids:
1642                         self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1643                 return
1644
1645
1646 class YoutubeUserIE(InfoExtractor):
1647         """Information Extractor for YouTube users."""
1648
1649         _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
1650         _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1651         _GDATA_PAGE_SIZE = 50
1652         _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
1653         _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
1654         IE_NAME = u'youtube:user'
1655
1656         def __init__(self, downloader=None):
1657                 InfoExtractor.__init__(self, downloader)
1658
1659         def report_download_page(self, username, start_index):
1660                 """Report attempt to download user page."""
1661                 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
1662                                 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
1663
1664         def _real_extract(self, url):
1665                 # Extract username
1666                 mobj = re.match(self._VALID_URL, url)
1667                 if mobj is None:
1668                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1669                         return
1670
1671                 username = mobj.group(1)
1672
1673                 # Download video ids using YouTube Data API. Result size per
1674                 # query is limited (currently to 50 videos) so we need to query
1675                 # page by page until there are no video ids - it means we got
1676                 # all of them.
1677
1678                 video_ids = []
1679                 pagenum = 0
1680
1681                 while True:
1682                         start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1683                         self.report_download_page(username, start_index)
1684
1685                         request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
1686
1687                         try:
1688                                 page = urllib2.urlopen(request).read()
1689                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1690                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1691                                 return
1692
1693                         # Extract video identifiers
1694                         ids_in_page = []
1695
1696                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1697                                 if mobj.group(1) not in ids_in_page:
1698                                         ids_in_page.append(mobj.group(1))
1699
1700                         video_ids.extend(ids_in_page)
1701
1702                         # A little optimization - if current page is not
1703                         # "full", ie. does not contain PAGE_SIZE video ids then
1704                         # we can assume that this page is the last one - there
1705                         # are no more ids on further pages - no need to query
1706                         # again.
1707
1708                         if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1709                                 break
1710
1711                         pagenum += 1
1712
1713                 all_ids_count = len(video_ids)
1714                 playliststart = self._downloader.params.get('playliststart', 1) - 1
1715                 playlistend = self._downloader.params.get('playlistend', -1)
1716
1717                 if playlistend == -1:
1718                         video_ids = video_ids[playliststart:]
1719                 else:
1720                         video_ids = video_ids[playliststart:playlistend]
1721
1722                 self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
1723                                 (username, all_ids_count, len(video_ids)))
1724
1725                 for video_id in video_ids:
1726                         self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
1727
1728
1729 class BlipTVUserIE(InfoExtractor):
1730         """Information Extractor for blip.tv users."""
1731
1732         _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
1733         _PAGE_SIZE = 12
1734         IE_NAME = u'blip.tv:user'
1735
1736         def __init__(self, downloader=None):
1737                 InfoExtractor.__init__(self, downloader)
1738
1739         def report_download_page(self, username, pagenum):
1740                 """Report attempt to download user page."""
1741                 self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
1742                                 (self.IE_NAME, username, pagenum))
1743
1744         def _real_extract(self, url):
1745                 # Extract username
1746                 mobj = re.match(self._VALID_URL, url)
1747                 if mobj is None:
1748                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1749                         return
1750
1751                 username = mobj.group(1)
1752
1753                 page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
1754
1755                 request = urllib2.Request(url)
1756
1757                 try:
1758                         page = urllib2.urlopen(request).read().decode('utf-8')
1759                         mobj = re.search(r'data-users-id="([^"]+)"', page)
1760                         page_base = page_base % mobj.group(1)
1761                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1762                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1763                         return
1764
1765
1766                 # Download video ids using BlipTV Ajax calls. Result size per
1767                 # query is limited (currently to 12 videos) so we need to query
1768                 # page by page until there are no video ids - it means we got
1769                 # all of them.
1770
1771                 video_ids = []
1772                 pagenum = 1
1773
1774                 while True:
1775                         self.report_download_page(username, pagenum)
1776
1777                         request = urllib2.Request( page_base + "&page=" + str(pagenum) )
1778
1779                         try:
1780                                 page = urllib2.urlopen(request).read().decode('utf-8')
1781                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1782                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1783                                 return
1784
1785                         # Extract video identifiers
1786                         ids_in_page = []
1787
1788                         for mobj in re.finditer(r'href="/([^"]+)"', page):
1789                                 if mobj.group(1) not in ids_in_page:
1790                                         ids_in_page.append(unescapeHTML(mobj.group(1)))
1791
1792                         video_ids.extend(ids_in_page)
1793
1794                         # A little optimization - if current page is not
1795                         # "full", ie. does not contain PAGE_SIZE video ids then
1796                         # we can assume that this page is the last one - there
1797                         # are no more ids on further pages - no need to query
1798                         # again.
1799
1800                         if len(ids_in_page) < self._PAGE_SIZE:
1801                                 break
1802
1803                         pagenum += 1
1804
1805                 all_ids_count = len(video_ids)
1806                 playliststart = self._downloader.params.get('playliststart', 1) - 1
1807                 playlistend = self._downloader.params.get('playlistend', -1)
1808
1809                 if playlistend == -1:
1810                         video_ids = video_ids[playliststart:]
1811                 else:
1812                         video_ids = video_ids[playliststart:playlistend]
1813
1814                 self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
1815                                 (self.IE_NAME, username, all_ids_count, len(video_ids)))
1816
1817                 for video_id in video_ids:
1818                         self._downloader.download([u'http://blip.tv/'+video_id])
1819
1820
1821 class DepositFilesIE(InfoExtractor):
1822         """Information extractor for depositfiles.com"""
1823
1824         _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
1825         IE_NAME = u'DepositFiles'
1826
1827         def __init__(self, downloader=None):
1828                 InfoExtractor.__init__(self, downloader)
1829
1830         def report_download_webpage(self, file_id):
1831                 """Report webpage download."""
1832                 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
1833
1834         def report_extraction(self, file_id):
1835                 """Report information extraction."""
1836                 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
1837
1838         def _real_extract(self, url):
1839                 file_id = url.split('/')[-1]
1840                 # Rebuild url in english locale
1841                 url = 'http://depositfiles.com/en/files/' + file_id
1842
1843                 # Retrieve file webpage with 'Free download' button pressed
1844                 free_download_indication = { 'gateway_result' : '1' }
1845                 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
1846                 try:
1847                         self.report_download_webpage(file_id)
1848                         webpage = urllib2.urlopen(request).read()
1849                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1850                         self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
1851                         return
1852
1853                 # Search for the real file URL
1854                 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
1855                 if (mobj is None) or (mobj.group(1) is None):
1856                         # Try to figure out reason of the error.
1857                         mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
1858                         if (mobj is not None) and (mobj.group(1) is not None):
1859                                 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
1860                                 self._downloader.trouble(u'ERROR: %s' % restriction_message)
1861                         else:
1862                                 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
1863                         return
1864
1865                 file_url = mobj.group(1)
1866                 file_extension = os.path.splitext(file_url)[1][1:]
1867
1868                 # Search for file title
1869                 mobj = re.search(r'<b title="(.*?)">', webpage)
1870                 if mobj is None:
1871                         self._downloader.trouble(u'ERROR: unable to extract title')
1872                         return
1873                 file_title = mobj.group(1).decode('utf-8')
1874
1875                 return [{
1876                         'id':           file_id.decode('utf-8'),
1877                         'url':          file_url.decode('utf-8'),
1878                         'uploader':     u'NA',
1879                         'upload_date':  u'NA',
1880                         'title':        file_title,
1881                         'ext':          file_extension.decode('utf-8'),
1882                         'format':       u'NA',
1883                         'player_url':   None,
1884                 }]
1885
1886
1887 class FacebookIE(InfoExtractor):
1888         """Information Extractor for Facebook"""
1889
1890         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
1891         _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
1892         _NETRC_MACHINE = 'facebook'
1893         _available_formats = ['video', 'highqual', 'lowqual']
1894         _video_extensions = {
1895                 'video': 'mp4',
1896                 'highqual': 'mp4',
1897                 'lowqual': 'mp4',
1898         }
1899         IE_NAME = u'facebook'
1900
1901         def __init__(self, downloader=None):
1902                 InfoExtractor.__init__(self, downloader)
1903
1904         def _reporter(self, message):
1905                 """Add header and report message."""
1906                 self._downloader.to_screen(u'[facebook] %s' % message)
1907
1908         def report_login(self):
1909                 """Report attempt to log in."""
1910                 self._reporter(u'Logging in')
1911
1912         def report_video_webpage_download(self, video_id):
1913                 """Report attempt to download video webpage."""
1914                 self._reporter(u'%s: Downloading video webpage' % video_id)
1915
1916         def report_information_extraction(self, video_id):
1917                 """Report attempt to extract video information."""
1918                 self._reporter(u'%s: Extracting video information' % video_id)
1919
1920         def _parse_page(self, video_webpage):
1921                 """Extract video information from page"""
1922                 # General data
1923                 data = {'title': r'\("video_title", "(.*?)"\)',
1924                         'description': r'<div class="datawrap">(.*?)</div>',
1925                         'owner': r'\("video_owner_name", "(.*?)"\)',
1926                         'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
1927                         }
1928                 video_info = {}
1929                 for piece in data.keys():
1930                         mobj = re.search(data[piece], video_webpage)
1931                         if mobj is not None:
1932                                 video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
1933
1934                 # Video urls
1935                 video_urls = {}
1936                 for fmt in self._available_formats:
1937                         mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
1938                         if mobj is not None:
1939                                 # URL is in a Javascript segment inside an escaped Unicode format within
1940                                 # the generally utf-8 page
1941                                 video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
1942                 video_info['video_urls'] = video_urls
1943
1944                 return video_info
1945
1946         def _real_initialize(self):
1947                 if self._downloader is None:
1948                         return
1949
1950                 useremail = None
1951                 password = None
1952                 downloader_params = self._downloader.params
1953
1954                 # Attempt to use provided username and password or .netrc data
1955                 if downloader_params.get('username', None) is not None:
1956                         useremail = downloader_params['username']
1957                         password = downloader_params['password']
1958                 elif downloader_params.get('usenetrc', False):
1959                         try:
1960                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
1961                                 if info is not None:
1962                                         useremail = info[0]
1963                                         password = info[2]
1964                                 else:
1965                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
1966                         except (IOError, netrc.NetrcParseError), err:
1967                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
1968                                 return
1969
1970                 if useremail is None:
1971                         return
1972
1973                 # Log in
1974                 login_form = {
1975                         'email': useremail,
1976                         'pass': password,
1977                         'login': 'Log+In'
1978                         }
1979                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
1980                 try:
1981                         self.report_login()
1982                         login_results = urllib2.urlopen(request).read()
1983                         if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
1984                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
1985                                 return
1986                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1987                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
1988                         return
1989
1990         def _real_extract(self, url):
1991                 mobj = re.match(self._VALID_URL, url)
1992                 if mobj is None:
1993                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1994                         return
1995                 video_id = mobj.group('ID')
1996
1997                 # Get video webpage
1998                 self.report_video_webpage_download(video_id)
1999                 request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2000                 try:
2001                         page = urllib2.urlopen(request)
2002                         video_webpage = page.read()
2003                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2004                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2005                         return
2006
2007                 # Start extracting information
2008                 self.report_information_extraction(video_id)
2009
2010                 # Extract information
2011                 video_info = self._parse_page(video_webpage)
2012
2013                 # uploader
2014                 if 'owner' not in video_info:
2015                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2016                         return
2017                 video_uploader = video_info['owner']
2018
2019                 # title
2020                 if 'title' not in video_info:
2021                         self._downloader.trouble(u'ERROR: unable to extract video title')
2022                         return
2023                 video_title = video_info['title']
2024                 video_title = video_title.decode('utf-8')
2025
2026                 # thumbnail image
2027                 if 'thumbnail' not in video_info:
2028                         self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2029                         video_thumbnail = ''
2030                 else:
2031                         video_thumbnail = video_info['thumbnail']
2032
2033                 # upload date
2034                 upload_date = u'NA'
2035                 if 'upload_date' in video_info:
2036                         upload_time = video_info['upload_date']
2037                         timetuple = email.utils.parsedate_tz(upload_time)
2038                         if timetuple is not None:
2039                                 try:
2040                                         upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2041                                 except:
2042                                         pass
2043
2044                 # description
2045                 video_description = video_info.get('description', 'No description available.')
2046
2047                 url_map = video_info['video_urls']
2048                 if len(url_map.keys()) > 0:
2049                         # Decide which formats to download
2050                         req_format = self._downloader.params.get('format', None)
2051                         format_limit = self._downloader.params.get('format_limit', None)
2052
2053                         if format_limit is not None and format_limit in self._available_formats:
2054                                 format_list = self._available_formats[self._available_formats.index(format_limit):]
2055                         else:
2056                                 format_list = self._available_formats
2057                         existing_formats = [x for x in format_list if x in url_map]
2058                         if len(existing_formats) == 0:
2059                                 self._downloader.trouble(u'ERROR: no known formats available for video')
2060                                 return
2061                         if req_format is None:
2062                                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2063                         elif req_format == 'worst':
2064                                 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
2065                         elif req_format == '-1':
2066                                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2067                         else:
2068                                 # Specific format
2069                                 if req_format not in url_map:
2070                                         self._downloader.trouble(u'ERROR: requested format not available')
2071                                         return
2072                                 video_url_list = [(req_format, url_map[req_format])] # Specific format
2073
2074                 results = []
2075                 for format_param, video_real_url in video_url_list:
2076                         # Extension
2077                         video_extension = self._video_extensions.get(format_param, 'mp4')
2078
2079                         results.append({
2080                                 'id':           video_id.decode('utf-8'),
2081                                 'url':          video_real_url.decode('utf-8'),
2082                                 'uploader':     video_uploader.decode('utf-8'),
2083                                 'upload_date':  upload_date,
2084                                 'title':        video_title,
2085                                 'ext':          video_extension.decode('utf-8'),
2086                                 'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
2087                                 'thumbnail':    video_thumbnail.decode('utf-8'),
2088                                 'description':  video_description.decode('utf-8'),
2089                                 'player_url':   None,
2090                         })
2091                 return results
2092
2093 class BlipTVIE(InfoExtractor):
2094         """Information extractor for blip.tv"""
2095
2096         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2097         _URL_EXT = r'^.*\.([a-z0-9]+)$'
2098         IE_NAME = u'blip.tv'
2099
2100         def report_extraction(self, file_id):
2101                 """Report information extraction."""
2102                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2103
2104         def report_direct_download(self, title):
2105                 """Report information extraction."""
2106                 self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
2107
2108         def _real_extract(self, url):
2109                 mobj = re.match(self._VALID_URL, url)
2110                 if mobj is None:
2111                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2112                         return
2113
2114                 if '?' in url:
2115                         cchar = '&'
2116                 else:
2117                         cchar = '?'
2118                 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2119                 request = urllib2.Request(json_url.encode('utf-8'))
2120                 self.report_extraction(mobj.group(1))
2121                 info = None
2122                 try:
2123                         urlh = urllib2.urlopen(request)
2124                         if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
2125                                 basename = url.split('/')[-1]
2126                                 title,ext = os.path.splitext(basename)
2127                                 title = title.decode('UTF-8')
2128                                 ext = ext.replace('.', '')
2129                                 self.report_direct_download(title)
2130                                 info = {
2131                                         'id': title,
2132                                         'url': url,
2133                                         'title': title,
2134                                         'ext': ext,
2135                                         'urlhandle': urlh
2136                                 }
2137                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2138                         self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
2139                         return
2140                 if info is None: # Regular URL
2141                         try:
2142                                 json_code = urlh.read()
2143                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2144                                 self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err))
2145                                 return
2146
2147                         try:
2148                                 json_data = json.loads(json_code)
2149                                 if 'Post' in json_data:
2150                                         data = json_data['Post']
2151                                 else:
2152                                         data = json_data
2153
2154                                 upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2155                                 video_url = data['media']['url']
2156                                 umobj = re.match(self._URL_EXT, video_url)
2157                                 if umobj is None:
2158                                         raise ValueError('Can not determine filename extension')
2159                                 ext = umobj.group(1)
2160
2161                                 info = {
2162                                         'id': data['item_id'],
2163                                         'url': video_url,
2164                                         'uploader': data['display_name'],
2165                                         'upload_date': upload_date,
2166                                         'title': data['title'],
2167                                         'ext': ext,
2168                                         'format': data['media']['mimeType'],
2169                                         'thumbnail': data['thumbnailUrl'],
2170                                         'description': data['description'],
2171                                         'player_url': data['embedUrl']
2172                                 }
2173                         except (ValueError,KeyError), err:
2174                                 self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2175                                 return
2176
2177                 std_headers['User-Agent'] = 'iTunes/10.6.1'
2178                 return [info]
2179
2180
2181 class MyVideoIE(InfoExtractor):
2182         """Information Extractor for myvideo.de."""
2183
2184         _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2185         IE_NAME = u'myvideo'
2186
2187         def __init__(self, downloader=None):
2188                 InfoExtractor.__init__(self, downloader)
2189
2190         def report_download_webpage(self, video_id):
2191                 """Report webpage download."""
2192                 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2193
2194         def report_extraction(self, video_id):
2195                 """Report information extraction."""
2196                 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2197
2198         def _real_extract(self,url):
2199                 mobj = re.match(self._VALID_URL, url)
2200                 if mobj is None:
2201                         self._download.trouble(u'ERROR: invalid URL: %s' % url)
2202                         return
2203
2204                 video_id = mobj.group(1)
2205
2206                 # Get video webpage
2207                 request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
2208                 try:
2209                         self.report_download_webpage(video_id)
2210                         webpage = urllib2.urlopen(request).read()
2211                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2212                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
2213                         return
2214
2215                 self.report_extraction(video_id)
2216                 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
2217                                  webpage)
2218                 if mobj is None:
2219                         self._downloader.trouble(u'ERROR: unable to extract media URL')
2220                         return
2221                 video_url = mobj.group(1) + ('/%s.flv' % video_id)
2222
2223                 mobj = re.search('<title>([^<]+)</title>', webpage)
2224                 if mobj is None:
2225                         self._downloader.trouble(u'ERROR: unable to extract title')
2226                         return
2227
2228                 video_title = mobj.group(1)
2229
2230                 return [{
2231                         'id':           video_id,
2232                         'url':          video_url,
2233                         'uploader':     u'NA',
2234                         'upload_date':  u'NA',
2235                         'title':        video_title,
2236                         'ext':          u'flv',
2237                         'format':       u'NA',
2238                         'player_url':   None,
2239                 }]
2240
2241 class ComedyCentralIE(InfoExtractor):
2242         """Information extractor for The Daily Show and Colbert Report """
2243
2244         _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
2245         IE_NAME = u'comedycentral'
2246
2247         def report_extraction(self, episode_id):
2248                 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
2249
2250         def report_config_download(self, episode_id):
2251                 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
2252
2253         def report_index_download(self, episode_id):
2254                 self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
2255
2256         def report_player_url(self, episode_id):
2257                 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
2258
2259         def _real_extract(self, url):
2260                 mobj = re.match(self._VALID_URL, url)
2261                 if mobj is None:
2262                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2263                         return
2264
2265                 if mobj.group('shortname'):
2266                         if mobj.group('shortname') in ('tds', 'thedailyshow'):
2267                                 url = u'http://www.thedailyshow.com/full-episodes/'
2268                         else:
2269                                 url = u'http://www.colbertnation.com/full-episodes/'
2270                         mobj = re.match(self._VALID_URL, url)
2271                         assert mobj is not None
2272
2273                 dlNewest = not mobj.group('episode')
2274                 if dlNewest:
2275                         epTitle = mobj.group('showname')
2276                 else:
2277                         epTitle = mobj.group('episode')
2278
2279                 req = urllib2.Request(url)
2280                 self.report_extraction(epTitle)
2281                 try:
2282                         htmlHandle = urllib2.urlopen(req)
2283                         html = htmlHandle.read()
2284                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2285                         self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
2286                         return
2287                 if dlNewest:
2288                         url = htmlHandle.geturl()
2289                         mobj = re.match(self._VALID_URL, url)
2290                         if mobj is None:
2291                                 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
2292                                 return
2293                         if mobj.group('episode') == '':
2294                                 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
2295                                 return
2296                         epTitle = mobj.group('episode')
2297
2298                 mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
2299                 if len(mMovieParams) == 0:
2300                         self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
2301                         return
2302
2303                 playerUrl_raw = mMovieParams[0][0]
2304                 self.report_player_url(epTitle)
2305                 try:
2306                         urlHandle = urllib2.urlopen(playerUrl_raw)
2307                         playerUrl = urlHandle.geturl()
2308                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2309                         self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
2310                         return
2311
2312                 uri = mMovieParams[0][1]
2313                 indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
2314                 self.report_index_download(epTitle)
2315                 try:
2316                         indexXml = urllib2.urlopen(indexUrl).read()
2317                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2318                         self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
2319                         return
2320
2321                 results = []
2322
2323                 idoc = xml.etree.ElementTree.fromstring(indexXml)
2324                 itemEls = idoc.findall('.//item')
2325                 for itemEl in itemEls:
2326                         mediaId = itemEl.findall('./guid')[0].text
2327                         shortMediaId = mediaId.split(':')[-1]
2328                         showId = mediaId.split(':')[-2].replace('.com', '')
2329                         officialTitle = itemEl.findall('./title')[0].text
2330                         officialDate = itemEl.findall('./pubDate')[0].text
2331
2332                         configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
2333                                                 urllib.urlencode({'uri': mediaId}))
2334                         configReq = urllib2.Request(configUrl)
2335                         self.report_config_download(epTitle)
2336                         try:
2337                                 configXml = urllib2.urlopen(configReq).read()
2338                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2339                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
2340                                 return
2341
2342                         cdoc = xml.etree.ElementTree.fromstring(configXml)
2343                         turls = []
2344                         for rendition in cdoc.findall('.//rendition'):
2345                                 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
2346                                 turls.append(finfo)
2347
2348                         if len(turls) == 0:
2349                                 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
2350                                 continue
2351
2352                         # For now, just pick the highest bitrate
2353                         format,video_url = turls[-1]
2354
2355                         effTitle = showId + u'-' + epTitle
2356                         info = {
2357                                 'id': shortMediaId,
2358                                 'url': video_url,
2359                                 'uploader': showId,
2360                                 'upload_date': officialDate,
2361                                 'title': effTitle,
2362                                 'ext': 'mp4',
2363                                 'format': format,
2364                                 'thumbnail': None,
2365                                 'description': officialTitle,
2366                                 'player_url': playerUrl
2367                         }
2368
2369                         results.append(info)
2370
2371                 return results
2372
2373
2374 class EscapistIE(InfoExtractor):
2375         """Information extractor for The Escapist """
2376
2377         _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
2378         IE_NAME = u'escapist'
2379
2380         def report_extraction(self, showName):
2381                 self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
2382
2383         def report_config_download(self, showName):
2384                 self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
2385
2386         def _real_extract(self, url):
2387                 mobj = re.match(self._VALID_URL, url)
2388                 if mobj is None:
2389                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2390                         return
2391                 showName = mobj.group('showname')
2392                 videoId = mobj.group('episode')
2393
2394                 self.report_extraction(showName)
2395                 try:
2396                         webPage = urllib2.urlopen(url)
2397                         webPageBytes = webPage.read()
2398                         m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
2399                         webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
2400                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2401                         self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
2402                         return
2403
2404                 descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
2405                 description = unescapeHTML(descMatch.group(1))
2406                 imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
2407                 imgUrl = unescapeHTML(imgMatch.group(1))
2408                 playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
2409                 playerUrl = unescapeHTML(playerUrlMatch.group(1))
2410                 configUrlMatch = re.search('config=(.*)$', playerUrl)
2411                 configUrl = urllib2.unquote(configUrlMatch.group(1))
2412
2413                 self.report_config_download(showName)
2414                 try:
2415                         configJSON = urllib2.urlopen(configUrl).read()
2416                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2417                         self._downloader.trouble(u'ERROR: unable to download configuration: ' + unicode(err))
2418                         return
2419
2420                 # Technically, it's JavaScript, not JSON
2421                 configJSON = configJSON.replace("'", '"')
2422
2423                 try:
2424                         config = json.loads(configJSON)
2425                 except (ValueError,), err:
2426                         self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + unicode(err))
2427                         return
2428
2429                 playlist = config['playlist']
2430                 videoUrl = playlist[1]['url']
2431
2432                 info = {
2433                         'id': videoId,
2434                         'url': videoUrl,
2435                         'uploader': showName,
2436                         'upload_date': None,
2437                         'title': showName,
2438                         'ext': 'flv',
2439                         'format': 'flv',
2440                         'thumbnail': imgUrl,
2441                         'description': description,
2442                         'player_url': playerUrl,
2443                 }
2444
2445                 return [info]
2446
2447
2448 class CollegeHumorIE(InfoExtractor):
2449         """Information extractor for collegehumor.com"""
2450
2451         _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
2452         IE_NAME = u'collegehumor'
2453
2454         def report_webpage(self, video_id):
2455                 """Report information extraction."""
2456                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2457
2458         def report_extraction(self, video_id):
2459                 """Report information extraction."""
2460                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2461
2462         def _real_extract(self, url):
2463                 mobj = re.match(self._VALID_URL, url)
2464                 if mobj is None:
2465                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2466                         return
2467                 video_id = mobj.group('videoid')
2468
2469                 self.report_webpage(video_id)
2470                 request = urllib2.Request(url)
2471                 try:
2472                         webpage = urllib2.urlopen(request).read()
2473                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2474                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2475                         return
2476
2477                 m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
2478                 if m is None:
2479                         self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
2480                         return
2481                 internal_video_id = m.group('internalvideoid')
2482
2483                 info = {
2484                         'id': video_id,
2485                         'internal_id': internal_video_id,
2486                 }
2487
2488                 self.report_extraction(video_id)
2489                 xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
2490                 try:
2491                         metaXml = urllib2.urlopen(xmlUrl).read()
2492                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2493                         self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
2494                         return
2495
2496                 mdoc = xml.etree.ElementTree.fromstring(metaXml)
2497                 try:
2498                         videoNode = mdoc.findall('./video')[0]
2499                         info['description'] = videoNode.findall('./description')[0].text
2500                         info['title'] = videoNode.findall('./caption')[0].text
2501                         info['url'] = videoNode.findall('./file')[0].text
2502                         info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
2503                         info['ext'] = info['url'].rpartition('.')[2]
2504                         info['format'] = info['ext']
2505                 except IndexError:
2506                         self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2507                         return
2508
2509                 return [info]
2510
2511
2512 class XVideosIE(InfoExtractor):
2513         """Information extractor for xvideos.com"""
2514
2515         _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
2516         IE_NAME = u'xvideos'
2517
2518         def report_webpage(self, video_id):
2519                 """Report information extraction."""
2520                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2521
2522         def report_extraction(self, video_id):
2523                 """Report information extraction."""
2524                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2525
2526         def _real_extract(self, url):
2527                 mobj = re.match(self._VALID_URL, url)
2528                 if mobj is None:
2529                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2530                         return
2531                 video_id = mobj.group(1).decode('utf-8')
2532
2533                 self.report_webpage(video_id)
2534
2535                 request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
2536                 try:
2537                         webpage = urllib2.urlopen(request).read()
2538                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2539                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2540                         return
2541
2542                 self.report_extraction(video_id)
2543
2544
2545                 # Extract video URL
2546                 mobj = re.search(r'flv_url=(.+?)&', webpage)
2547                 if mobj is None:
2548                         self._downloader.trouble(u'ERROR: unable to extract video url')
2549                         return
2550                 video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
2551
2552
2553                 # Extract title
2554                 mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
2555                 if mobj is None:
2556                         self._downloader.trouble(u'ERROR: unable to extract video title')
2557                         return
2558                 video_title = mobj.group(1).decode('utf-8')
2559
2560
2561                 # Extract video thumbnail
2562                 mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
2563                 if mobj is None:
2564                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
2565                         return
2566                 video_thumbnail = mobj.group(0).decode('utf-8')
2567
2568                 info = {
2569                         'id': video_id,
2570                         'url': video_url,
2571                         'uploader': None,
2572                         'upload_date': None,
2573                         'title': video_title,
2574                         'ext': 'flv',
2575                         'format': 'flv',
2576                         'thumbnail': video_thumbnail,
2577                         'description': None,
2578                         'player_url': None,
2579                 }
2580
2581                 return [info]
2582
2583
2584 class SoundcloudIE(InfoExtractor):
2585         """Information extractor for soundcloud.com
2586            To access the media, the uid of the song and a stream token
2587            must be extracted from the page source and the script must make
2588            a request to media.soundcloud.com/crossdomain.xml. Then
2589            the media can be grabbed by requesting from an url composed
2590            of the stream token and uid
2591          """
2592
2593         _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
2594         IE_NAME = u'soundcloud'
2595
2596         def __init__(self, downloader=None):
2597                 InfoExtractor.__init__(self, downloader)
2598
2599         def report_webpage(self, video_id):
2600                 """Report information extraction."""
2601                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2602
2603         def report_extraction(self, video_id):
2604                 """Report information extraction."""
2605                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2606
2607         def _real_extract(self, url):
2608                 mobj = re.match(self._VALID_URL, url)
2609                 if mobj is None:
2610                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2611                         return
2612
2613                 # extract uploader (which is in the url)
2614                 uploader = mobj.group(1).decode('utf-8')
2615                 # extract simple title (uploader + slug of song title)
2616                 slug_title =  mobj.group(2).decode('utf-8')
2617                 simple_title = uploader + u'-' + slug_title
2618
2619                 self.report_webpage('%s/%s' % (uploader, slug_title))
2620
2621                 request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
2622                 try:
2623                         webpage = urllib2.urlopen(request).read()
2624                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2625                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2626                         return
2627
2628                 self.report_extraction('%s/%s' % (uploader, slug_title))
2629
2630                 # extract uid and stream token that soundcloud hands out for access
2631                 mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
2632                 if mobj:
2633                         video_id = mobj.group(1)
2634                         stream_token = mobj.group(2)
2635
2636                 # extract unsimplified title
2637                 mobj = re.search('"title":"(.*?)",', webpage)
2638                 if mobj:
2639                         title = mobj.group(1).decode('utf-8')
2640                 else:
2641                         title = simple_title
2642
2643                 # construct media url (with uid/token)
2644                 mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
2645                 mediaURL = mediaURL % (video_id, stream_token)
2646
2647                 # description
2648                 description = u'No description available'
2649                 mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
2650                 if mobj:
2651                         description = mobj.group(1)
2652
2653                 # upload date
2654                 upload_date = None
2655                 mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
2656                 if mobj:
2657                         try:
2658                                 upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
2659                         except Exception, e:
2660                                 self._downloader.to_stderr(str(e))
2661
2662                 # for soundcloud, a request to a cross domain is required for cookies
2663                 request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
2664
2665                 return [{
2666                         'id':           video_id.decode('utf-8'),
2667                         'url':          mediaURL,
2668                         'uploader':     uploader.decode('utf-8'),
2669                         'upload_date':  upload_date,
2670                         'title':        title,
2671                         'ext':          u'mp3',
2672                         'format':       u'NA',
2673                         'player_url':   None,
2674                         'description': description.decode('utf-8')
2675                 }]
2676
2677
2678 class InfoQIE(InfoExtractor):
2679         """Information extractor for infoq.com"""
2680
2681         _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
2682         IE_NAME = u'infoq'
2683
2684         def report_webpage(self, video_id):
2685                 """Report information extraction."""
2686                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2687
2688         def report_extraction(self, video_id):
2689                 """Report information extraction."""
2690                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2691
2692         def _real_extract(self, url):
2693                 mobj = re.match(self._VALID_URL, url)
2694                 if mobj is None:
2695                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2696                         return
2697
2698                 self.report_webpage(url)
2699
2700                 request = urllib2.Request(url)
2701                 try:
2702                         webpage = urllib2.urlopen(request).read()
2703                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2704                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2705                         return
2706
2707                 self.report_extraction(url)
2708
2709
2710                 # Extract video URL
2711                 mobj = re.search(r"jsclassref='([^']*)'", webpage)
2712                 if mobj is None:
2713                         self._downloader.trouble(u'ERROR: unable to extract video url')
2714                         return
2715                 video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
2716
2717
2718                 # Extract title
2719                 mobj = re.search(r'contentTitle = "(.*?)";', webpage)
2720                 if mobj is None:
2721                         self._downloader.trouble(u'ERROR: unable to extract video title')
2722                         return
2723                 video_title = mobj.group(1).decode('utf-8')
2724
2725                 # Extract description
2726                 video_description = u'No description available.'
2727                 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
2728                 if mobj is not None:
2729                         video_description = mobj.group(1).decode('utf-8')
2730
2731                 video_filename = video_url.split('/')[-1]
2732                 video_id, extension = video_filename.split('.')
2733
2734                 info = {
2735                         'id': video_id,
2736                         'url': video_url,
2737                         'uploader': None,
2738                         'upload_date': None,
2739                         'title': video_title,
2740                         'ext': extension,
2741                         'format': extension, # Extension is always(?) mp4, but seems to be flv
2742                         'thumbnail': None,
2743                         'description': video_description,
2744                         'player_url': None,
2745                 }
2746
2747                 return [info]
2748
2749 class MixcloudIE(InfoExtractor):
2750         """Information extractor for www.mixcloud.com"""
2751         _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
2752         IE_NAME = u'mixcloud'
2753
2754         def __init__(self, downloader=None):
2755                 InfoExtractor.__init__(self, downloader)
2756
2757         def report_download_json(self, file_id):
2758                 """Report JSON download."""
2759                 self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
2760
2761         def report_extraction(self, file_id):
2762                 """Report information extraction."""
2763                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2764
2765         def get_urls(self, jsonData, fmt, bitrate='best'):
2766                 """Get urls from 'audio_formats' section in json"""
2767                 file_url = None
2768                 try:
2769                         bitrate_list = jsonData[fmt]
2770                         if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
2771                                 bitrate = max(bitrate_list) # select highest
2772
2773                         url_list = jsonData[fmt][bitrate]
2774                 except TypeError: # we have no bitrate info.
2775                         url_list = jsonData[fmt]
2776                 return url_list
2777
2778         def check_urls(self, url_list):
2779                 """Returns 1st active url from list"""
2780                 for url in url_list:
2781                         try:
2782                                 urllib2.urlopen(url)
2783                                 return url
2784                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2785                                 url = None
2786
2787                 return None
2788
2789         def _print_formats(self, formats):
2790                 print 'Available formats:'
2791                 for fmt in formats.keys():
2792                         for b in formats[fmt]:
2793                                 try:
2794                                         ext = formats[fmt][b][0]
2795                                         print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])
2796                                 except TypeError: # we have no bitrate info
2797                                         ext = formats[fmt][0]
2798                                         print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])
2799                                         break
2800
2801         def _real_extract(self, url):
2802                 mobj = re.match(self._VALID_URL, url)
2803                 if mobj is None:
2804                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2805                         return
2806                 # extract uploader & filename from url
2807                 uploader = mobj.group(1).decode('utf-8')
2808                 file_id = uploader + "-" + mobj.group(2).decode('utf-8')
2809
2810                 # construct API request
2811                 file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
2812                 # retrieve .json file with links to files
2813                 request = urllib2.Request(file_url)
2814                 try:
2815                         self.report_download_json(file_url)
2816                         jsonData = urllib2.urlopen(request).read()
2817                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2818                         self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err))
2819                         return
2820
2821                 # parse JSON
2822                 json_data = json.loads(jsonData)
2823                 player_url = json_data['player_swf_url']
2824                 formats = dict(json_data['audio_formats'])
2825
2826                 req_format = self._downloader.params.get('format', None)
2827                 bitrate = None
2828
2829                 if self._downloader.params.get('listformats', None):
2830                         self._print_formats(formats)
2831                         return
2832
2833                 if req_format is None or req_format == 'best':
2834                         for format_param in formats.keys():
2835                                 url_list = self.get_urls(formats, format_param)
2836                                 # check urls
2837                                 file_url = self.check_urls(url_list)
2838                                 if file_url is not None:
2839                                         break # got it!
2840                 else:
2841                         if req_format not in formats.keys():
2842                                 self._downloader.trouble(u'ERROR: format is not available')
2843                                 return
2844
2845                         url_list = self.get_urls(formats, req_format)
2846                         file_url = self.check_urls(url_list)
2847                         format_param = req_format
2848
2849                 return [{
2850                         'id': file_id.decode('utf-8'),
2851                         'url': file_url.decode('utf-8'),
2852                         'uploader':     uploader.decode('utf-8'),
2853                         'upload_date': u'NA',
2854                         'title': json_data['name'],
2855                         'ext': file_url.split('.')[-1].decode('utf-8'),
2856                         'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
2857                         'thumbnail': json_data['thumbnail_url'],
2858                         'description': json_data['description'],
2859                         'player_url': player_url.decode('utf-8'),
2860                 }]
2861
2862 class StanfordOpenClassroomIE(InfoExtractor):
2863         """Information extractor for Stanford's Open ClassRoom"""
2864
2865         _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
2866         IE_NAME = u'stanfordoc'
2867
2868         def report_download_webpage(self, objid):
2869                 """Report information extraction."""
2870                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
2871
2872         def report_extraction(self, video_id):
2873                 """Report information extraction."""
2874                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2875
2876         def _real_extract(self, url):
2877                 mobj = re.match(self._VALID_URL, url)
2878                 if mobj is None:
2879                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2880                         return
2881
2882                 if mobj.group('course') and mobj.group('video'): # A specific video
2883                         course = mobj.group('course')
2884                         video = mobj.group('video')
2885                         info = {
2886                                 'id': course + '_' + video,
2887                         }
2888
2889                         self.report_extraction(info['id'])
2890                         baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
2891                         xmlUrl = baseUrl + video + '.xml'
2892                         try:
2893                                 metaXml = urllib2.urlopen(xmlUrl).read()
2894                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2895                                 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
2896                                 return
2897                         mdoc = xml.etree.ElementTree.fromstring(metaXml)
2898                         try:
2899                                 info['title'] = mdoc.findall('./title')[0].text
2900                                 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
2901                         except IndexError:
2902                                 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2903                                 return
2904                         info['ext'] = info['url'].rpartition('.')[2]
2905                         info['format'] = info['ext']
2906                         return [info]
2907                 elif mobj.group('course'): # A course page
2908                         course = mobj.group('course')
2909                         info = {
2910                                 'id': course,
2911                                 'type': 'playlist',
2912                         }
2913
2914                         self.report_download_webpage(info['id'])
2915                         try:
2916                                 coursepage = urllib2.urlopen(url).read()
2917                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2918                                 self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
2919                                 return
2920
2921                         m = re.search('<h1>([^<]+)</h1>', coursepage)
2922                         if m:
2923                                 info['title'] = unescapeHTML(m.group(1))
2924                         else:
2925                                 info['title'] = info['id']
2926
2927                         m = re.search('<description>([^<]+)</description>', coursepage)
2928                         if m:
2929                                 info['description'] = unescapeHTML(m.group(1))
2930
2931                         links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
2932                         info['list'] = [
2933                                 {
2934                                         'type': 'reference',
2935                                         'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
2936                                 }
2937                                         for vpage in links]
2938                         results = []
2939                         for entry in info['list']:
2940                                 assert entry['type'] == 'reference'
2941                                 results += self.extract(entry['url'])
2942                         return results
2943
2944                 else: # Root page
2945                         info = {
2946                                 'id': 'Stanford OpenClassroom',
2947                                 'type': 'playlist',
2948                         }
2949
2950                         self.report_download_webpage(info['id'])
2951                         rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
2952                         try:
2953                                 rootpage = urllib2.urlopen(rootURL).read()
2954                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2955                                 self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
2956                                 return
2957
2958                         info['title'] = info['id']
2959
2960                         links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
2961                         info['list'] = [
2962                                 {
2963                                         'type': 'reference',
2964                                         'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
2965                                 }
2966                                         for cpage in links]
2967
2968                         results = []
2969                         for entry in info['list']:
2970                                 assert entry['type'] == 'reference'
2971                                 results += self.extract(entry['url'])
2972                         return results
2973
2974 class MTVIE(InfoExtractor):
2975         """Information extractor for MTV.com"""
2976
2977         _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
2978         IE_NAME = u'mtv'
2979
2980         def report_webpage(self, video_id):
2981                 """Report information extraction."""
2982                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2983
2984         def report_extraction(self, video_id):
2985                 """Report information extraction."""
2986                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2987
2988         def _real_extract(self, url):
2989                 mobj = re.match(self._VALID_URL, url)
2990                 if mobj is None:
2991                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2992                         return
2993                 if not mobj.group('proto'):
2994                         url = 'http://' + url
2995                 video_id = mobj.group('videoid')
2996                 self.report_webpage(video_id)
2997
2998                 request = urllib2.Request(url)
2999                 try:
3000                         webpage = urllib2.urlopen(request).read()
3001                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3002                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
3003                         return
3004
3005                 mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
3006                 if mobj is None:
3007                         self._downloader.trouble(u'ERROR: unable to extract song name')
3008                         return
3009                 song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3010                 mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
3011                 if mobj is None:
3012                         self._downloader.trouble(u'ERROR: unable to extract performer')
3013                         return
3014                 performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3015                 video_title = performer + ' - ' + song_name
3016
3017                 mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
3018                 if mobj is None:
3019                         self._downloader.trouble(u'ERROR: unable to mtvn_uri')
3020                         return
3021                 mtvn_uri = mobj.group(1)
3022
3023                 mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
3024                 if mobj is None:
3025                         self._downloader.trouble(u'ERROR: unable to extract content id')
3026                         return
3027                 content_id = mobj.group(1)
3028
3029                 videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
3030                 self.report_extraction(video_id)
3031                 request = urllib2.Request(videogen_url)
3032                 try:
3033                         metadataXml = urllib2.urlopen(request).read()
3034                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3035                         self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err))
3036                         return
3037
3038                 mdoc = xml.etree.ElementTree.fromstring(metadataXml)
3039                 renditions = mdoc.findall('.//rendition')
3040
3041                 # For now, always pick the highest quality.
3042                 rendition = renditions[-1]
3043
3044                 try:
3045                         _,_,ext = rendition.attrib['type'].partition('/')
3046                         format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
3047                         video_url = rendition.find('./src').text
3048                 except KeyError:
3049                         self._downloader.trouble('Invalid rendition field.')
3050                         return
3051
3052                 info = {
3053                         'id': video_id,
3054                         'url': video_url,
3055                         'uploader': performer,
3056                         'title': video_title,
3057                         'ext': ext,
3058                         'format': format,
3059                 }
3060
3061                 return [info]
3062
3063
3064 class YoukuIE(InfoExtractor):
3065
3066         _VALID_URL =  r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
3067         IE_NAME = u'Youku'
3068
3069         def __init__(self, downloader=None):
3070                 InfoExtractor.__init__(self, downloader)
3071
3072         def report_download_webpage(self, file_id):
3073                 """Report webpage download."""
3074                 self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id)
3075
3076         def report_extraction(self, file_id):
3077                 """Report information extraction."""
3078                 self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id)
3079
3080         def _gen_sid(self):
3081                 nowTime = int(time.time() * 1000)
3082                 random1 = random.randint(1000,1998)
3083                 random2 = random.randint(1000,9999)
3084
3085                 return "%d%d%d" %(nowTime,random1,random2)
3086
3087         def _get_file_ID_mix_string(self, seed):
3088                 mixed = []
3089                 source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
3090                 seed = float(seed)
3091                 for i in range(len(source)):
3092                         seed  =  (seed * 211 + 30031 ) % 65536
3093                         index  =  math.floor(seed / 65536 * len(source) )
3094                         mixed.append(source[int(index)])
3095                         source.remove(source[int(index)])
3096                 #return ''.join(mixed)
3097                 return mixed
3098
3099         def _get_file_id(self, fileId, seed):
3100                 mixed = self._get_file_ID_mix_string(seed)
3101                 ids = fileId.split('*')
3102                 realId = []
3103                 for ch in ids:
3104                         if ch:
3105                                 realId.append(mixed[int(ch)])
3106                 return ''.join(realId)
3107
3108         def _real_extract(self, url):
3109                 mobj = re.match(self._VALID_URL, url)
3110                 if mobj is None:
3111                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3112                         return
3113                 video_id = mobj.group('ID')
3114
3115                 info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
3116
3117                 request = urllib2.Request(info_url, None, std_headers)
3118                 try:
3119                         self.report_download_webpage(video_id)
3120                         jsondata = urllib2.urlopen(request).read()
3121                 except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
3122                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
3123                         return
3124
3125                 self.report_extraction(video_id)
3126                 try:
3127                         config = json.loads(jsondata)
3128
3129                         video_title =  config['data'][0]['title']
3130                         seed = config['data'][0]['seed']
3131
3132                         format = self._downloader.params.get('format', None)
3133                         supported_format = config['data'][0]['streamfileids'].keys()
3134
3135                         if format is None or format == 'best':
3136                                 if 'hd2' in supported_format:
3137                                         format = 'hd2'
3138                                 else:
3139                                         format = 'flv'
3140                                 ext = u'flv'
3141                         elif format == 'worst':
3142                                 format = 'mp4'
3143                                 ext = u'mp4'
3144                         else:
3145                                 format = 'flv'
3146                                 ext = u'flv'
3147
3148
3149                         fileid = config['data'][0]['streamfileids'][format]
3150                         seg_number = len(config['data'][0]['segs'][format])
3151
3152                         keys=[]
3153                         for i in xrange(seg_number):
3154                                 keys.append(config['data'][0]['segs'][format][i]['k'])
3155
3156                         #TODO check error
3157                         #youku only could be viewed from mainland china
3158                 except:
3159                         self._downloader.trouble(u'ERROR: unable to extract info section')
3160                         return
3161
3162                 files_info=[]
3163                 sid = self._gen_sid()
3164                 fileid = self._get_file_id(fileid, seed)
3165
3166                 #column 8,9 of fileid represent the segment number
3167                 #fileid[7:9] should be changed
3168                 for index, key in enumerate(keys):
3169
3170                         temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
3171                         download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
3172
3173                         info = {
3174                                 'id': '%s_part%02d' % (video_id, index),
3175                                 'url': download_url,
3176                                 'uploader': None,
3177                                 'title': video_title,
3178                                 'ext': ext,
3179                                 'format': u'NA'
3180                         }
3181                         files_info.append(info)
3182
3183                 return files_info
3184
3185
3186 class XNXXIE(InfoExtractor):
3187         """Information extractor for xnxx.com"""
3188
3189         _VALID_URL = r'^(?:https?://)?video\.xnxx\.com/video([0-9]+)/(.*)'
3190         IE_NAME = u'xnxx'
3191         VIDEO_URL_RE = r'flv_url=(.*?)&amp;'
3192         VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
3193         VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'
3194
3195         def report_webpage(self, video_id):
3196                 """Report information extraction"""
3197                 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3198
3199         def report_extraction(self, video_id):
3200                 """Report information extraction"""
3201                 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3202
3203         def _real_extract(self, url):
3204                 mobj = re.match(self._VALID_URL, url)
3205                 if mobj is None:
3206                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3207                         return
3208                 video_id = mobj.group(1).decode('utf-8')
3209
3210                 self.report_webpage(video_id)
3211
3212                 # Get webpage content
3213                 try:
3214                         webpage = urllib2.urlopen(url).read()
3215                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3216                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3217                         return
3218
3219                 result = re.search(self.VIDEO_URL_RE, webpage)
3220                 if result is None:
3221                         self._downloader.trouble(u'ERROR: unable to extract video url')
3222                         return
3223                 video_url = urllib.unquote(result.group(1).decode('utf-8'))
3224
3225                 result = re.search(self.VIDEO_TITLE_RE, webpage)
3226                 if result is None:
3227                         self._downloader.trouble(u'ERROR: unable to extract video title')
3228                         return
3229                 video_title = result.group(1).decode('utf-8')
3230
3231                 result = re.search(self.VIDEO_THUMB_RE, webpage)
3232                 if result is None:
3233                         self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
3234                         return
3235                 video_thumbnail = result.group(1).decode('utf-8')
3236
3237                 info = {'id': video_id,
3238                                 'url': video_url,
3239                                 'uploader': None,
3240                                 'upload_date': None,
3241                                 'title': video_title,
3242                                 'ext': 'flv',
3243                                 'format': 'flv',
3244                                 'thumbnail': video_thumbnail,
3245                                 'description': None,
3246                                 'player_url': None}
3247
3248                 return [info]
3249
3250
3251 class GooglePlusIE(InfoExtractor):
3252         """Information extractor for plus.google.com."""
3253
3254         _VALID_URL = r'(?:https://)?plus\.google\.com/(?:\w+/)*?(\d+)/posts/(\w+)'
3255         IE_NAME = u'plus.google'
3256
3257         def __init__(self, downloader=None):
3258                 InfoExtractor.__init__(self, downloader)
3259
3260         def report_extract_entry(self, url):
3261                 """Report downloading extry"""
3262                 self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url.decode('utf-8'))
3263
3264         def report_date(self, upload_date):
3265                 """Report downloading extry"""
3266                 self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
3267
3268         def report_uploader(self, uploader):
3269                 """Report downloading extry"""
3270                 self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader.decode('utf-8'))
3271
3272         def report_title(self, video_title):
3273                 """Report downloading extry"""
3274                 self._downloader.to_screen(u'[plus.google] Title: %s' % video_title.decode('utf-8'))
3275
3276         def report_extract_vid_page(self, video_page):
3277                 """Report information extraction."""
3278                 self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page.decode('utf-8'))
3279
3280         def _real_extract(self, url):
3281                 # Extract id from URL
3282                 mobj = re.match(self._VALID_URL, url)
3283                 if mobj is None:
3284                         self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
3285                         return
3286
3287                 post_url = mobj.group(0)
3288                 video_id = mobj.group(2)
3289
3290                 video_extension = 'flv'
3291
3292                 # Step 1, Retrieve post webpage to extract further information
3293                 self.report_extract_entry(post_url)
3294                 request = urllib2.Request(post_url)
3295                 try:
3296                         webpage = urllib2.urlopen(request).read()
3297                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3298                         self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % str(err))
3299                         return
3300
3301                 # Extract update date
3302                 upload_date = u'NA'
3303                 pattern = 'title="Timestamp">(.*?)</a>'
3304                 mobj = re.search(pattern, webpage)
3305                 if mobj:
3306                         upload_date = mobj.group(1)
3307                         # Convert timestring to a format suitable for filename
3308                         upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
3309                         upload_date = upload_date.strftime('%Y%m%d')
3310                 self.report_date(upload_date)
3311
3312                 # Extract uploader
3313                 uploader = u'NA'
3314                 pattern = r'rel\="author".*?>(.*?)</a>'
3315                 mobj = re.search(pattern, webpage)
3316                 if mobj:
3317                         uploader = mobj.group(1)
3318                 self.report_uploader(uploader)
3319
3320                 # Extract title
3321                 # Get the first line for title
3322                 video_title = u'NA'
3323                 pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
3324                 mobj = re.search(pattern, webpage)
3325                 if mobj:
3326                         video_title = mobj.group(1)
3327                 self.report_title(video_title)
3328
3329                 # Step 2, Stimulate clicking the image box to launch video
3330                 pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
3331                 mobj = re.search(pattern, webpage)
3332                 if mobj is None:
3333                         self._downloader.trouble(u'ERROR: unable to extract video page URL')
3334
3335                 video_page = mobj.group(1)
3336                 request = urllib2.Request(video_page)
3337                 try:
3338                         webpage = urllib2.urlopen(request).read()
3339                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3340                         self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
3341                         return
3342                 self.report_extract_vid_page(video_page)
3343
3344
3345                 # Extract video links on video page
3346                 """Extract video links of all sizes"""
3347                 pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
3348                 mobj = re.findall(pattern, webpage)
3349                 if len(mobj) == 0:
3350                         self._downloader.trouble(u'ERROR: unable to extract video links')
3351
3352                 # Sort in resolution
3353                 links = sorted(mobj)
3354
3355                 # Choose the lowest of the sort, i.e. highest resolution
3356                 video_url = links[-1]
3357                 # Only get the url. The resolution part in the tuple has no use anymore
3358                 video_url = video_url[-1]
3359                 # Treat escaped \u0026 style hex
3360                 video_url = unicode(video_url, "unicode_escape")
3361
3362
3363                 return [{
3364                         'id':           video_id.decode('utf-8'),
3365                         'url':          video_url,
3366                         'uploader':     uploader.decode('utf-8'),
3367                         'upload_date':  upload_date.decode('utf-8'),
3368                         'title':        video_title.decode('utf-8'),
3369                         'ext':          video_extension.decode('utf-8'),
3370                         'format':       u'NA',
3371                         'player_url':   None,
3372                 }]
3373
3374
3375
3376 class YouPornIE(InfoExtractor):
3377         """Information extractor for youporn.com."""
3378
3379         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+)'
3380         IE_NAME = u'youporn'
3381         VIDEO_TITLE_RE = r'videoTitleArea">(?P<title>.*)</h1>'
3382         VIDEO_DATE_RE = r'Date:</b>(?P<date>.*)</li>'
3383         VIDEO_UPLOADER_RE = r'Submitted:</b>(?P<uploader>.*)</li>'
3384         DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
3385         LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">'
3386
3387         def __init__(self, downloader=None):
3388                 InfoExtractor.__init__(self, downloader)
3389
3390         def report_id(self, video_id):
3391                 """Report finding video ID"""
3392                 self._downloader.to_screen(u'[youporn] Video ID: %s' % video_id)
3393
3394         def report_webpage(self, url):
3395                 """Report downloading page"""
3396                 self._downloader.to_screen(u'[youporn] Downloaded page: %s' % url)
3397
3398         def report_title(self, video_title):
3399                 """Report dfinding title"""
3400                 self._downloader.to_screen(u'[youporn] Title: %s' % video_title)
3401
3402         def report_uploader(self, uploader):
3403                 """Report dfinding title"""
3404                 self._downloader.to_screen(u'[youporn] Uploader: %s' % uploader)
3405
3406         def report_upload_date(self, video_date):
3407                 """Report finding date"""
3408                 self._downloader.to_screen(u'[youporn] Date: %s' % video_date)
3409
3410         def _print_formats(self, formats):
3411                 """Print all available formats"""
3412                 print 'Available formats:'
3413                 print u'ext\t\tformat'
3414                 print u'---------------------------------'
3415                 for format in formats:
3416                         print u'%s\t\t%s'  % (format['ext'], format['format'])
3417
3418         def _specific(self, req_format, formats):
3419                 for x in formats:
3420                         if(x["format"]==req_format):
3421                                 return x
3422                 return None
3423
3424
3425         def _real_extract(self, url):
3426                 mobj = re.match(self._VALID_URL, url)
3427                 if mobj is None:
3428                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3429                         return
3430
3431                 video_id = mobj.group('videoid').decode('utf-8')
3432                 self.report_id(video_id)
3433
3434                 # Get webpage content
3435                 try:
3436                         webpage = urllib2.urlopen(url).read()
3437                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3438                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3439                         return
3440                 self.report_webpage(url)
3441
3442                 # Get the video title
3443                 result = re.search(self.VIDEO_TITLE_RE, webpage)
3444                 if result is None:
3445                         self._downloader.trouble(u'ERROR: unable to extract video title')
3446                         return
3447                 video_title = result.group('title').decode('utf-8').strip()
3448                 self.report_title(video_title)
3449
3450                 # Get the video date
3451                 result = re.search(self.VIDEO_DATE_RE, webpage)
3452                 if result is None:
3453                         self._downloader.trouble(u'ERROR: unable to extract video date')
3454                         return
3455                 upload_date = result.group('date').decode('utf-8').strip()
3456                 self.report_upload_date(upload_date)
3457
3458                 # Get the video uploader
3459                 result = re.search(self.VIDEO_UPLOADER_RE, webpage)
3460                 if result is None:
3461                         self._downloader.trouble(u'ERROR: unable to extract uploader')
3462                         return
3463                 video_uploader = result.group('uploader').decode('utf-8').strip()
3464                 video_uploader = clean_html( video_uploader )
3465                 self.report_uploader(video_uploader)
3466
3467                 # Get all of the formats available
3468                 result = re.search(self.DOWNLOAD_LIST_RE, webpage)
3469                 if result is None:
3470                         self._downloader.trouble(u'ERROR: unable to extract download list')
3471                         return
3472                 download_list_html = result.group('download_list').decode('utf-8').strip()
3473
3474                 # Get all of the links from the page
3475                 links = re.findall(self.LINK_RE, download_list_html)
3476                 if(len(links) == 0):
3477                         self._downloader.trouble(u'ERROR: no known formats available for video')
3478                         return
3479
3480                 self._downloader.to_screen(u'[youporn] Links found: %d' % len(links))
3481
3482                 formats = []
3483                 for link in links:
3484
3485                         # A link looks like this:
3486                         # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
3487                         # A path looks like this:
3488                         # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
3489                         video_url = unescapeHTML( link.decode('utf-8') )
3490                         path = urlparse( video_url ).path
3491                         extension = os.path.splitext( path )[1][1:]
3492                         format = path.split('/')[4].split('_')[:2]
3493                         size = format[0]
3494                         bitrate = format[1]
3495                         format = "-".join( format )
3496                         title = u'%s-%s-%s' % (video_title, size, bitrate)
3497
3498                         formats.append({
3499                                 'id': video_id,
3500                                 'url': video_url,
3501                                 'uploader': video_uploader,
3502                                 'upload_date': upload_date,
3503                                 'title': title,
3504                                 'ext': extension,
3505                                 'format': format,
3506                                 'thumbnail': None,
3507                                 'description': None,
3508                                 'player_url': None
3509                         })
3510
3511                 if self._downloader.params.get('listformats', None):
3512                         self._print_formats(formats)
3513                         return
3514
3515                 req_format = self._downloader.params.get('format', None)
3516                 #format_limit = self._downloader.params.get('format_limit', None)
3517                 self._downloader.to_screen(u'[youporn] Format: %s' % req_format)
3518
3519
3520                 if req_format is None or req_format == 'best':
3521                         return [formats[0]]
3522                 elif req_format == 'worst':
3523                         return [formats[-1]]
3524                 elif req_format in ('-1', 'all'):
3525                         return formats
3526                 else:
3527                         format = self._specific( req_format, formats )
3528                         if result is None:
3529                                 self._downloader.trouble(u'ERROR: requested format not available')
3530                                 return
3531                         return [format]
3532
3533
3534
3535
3536 class PornotubeIE(InfoExtractor):
3537         """Information extractor for pornotube.com."""
3538
3539         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
3540         IE_NAME = u'pornotube'
3541         VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",'
3542         VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
3543
3544
3545         def __init__(self, downloader=None):
3546                 InfoExtractor.__init__(self, downloader)
3547
3548         def report_extract_entry(self, url):
3549                 """Report downloading extry"""
3550                 self._downloader.to_screen(u'[pornotube] Downloading entry: %s' % url.decode('utf-8'))
3551
3552         def report_date(self, upload_date):
3553                 """Report finding uploaded date"""
3554                 self._downloader.to_screen(u'[pornotube] Entry date: %s' % upload_date)
3555
3556         def report_webpage(self, url):
3557                 """Report downloading page"""
3558                 self._downloader.to_screen(u'[pornotube] Downloaded page: %s' % url)
3559
3560         def report_title(self, video_title):
3561                 """Report downloading extry"""
3562                 self._downloader.to_screen(u'[pornotube] Title: %s' % video_title.decode('utf-8'))
3563
3564         def _real_extract(self, url):
3565                 mobj = re.match(self._VALID_URL, url)
3566                 if mobj is None:
3567                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3568                         return
3569
3570                 video_id = mobj.group('videoid').decode('utf-8')
3571                 video_title = mobj.group('title').decode('utf-8')
3572                 self.report_title(video_title);
3573
3574                 # Get webpage content
3575                 try:
3576                         webpage = urllib2.urlopen(url).read()
3577                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3578                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3579                         return
3580                 self.report_webpage(url)
3581
3582                 # Get the video URL
3583                 result = re.search(self.VIDEO_URL_RE, webpage)
3584                 if result is None:
3585                         self._downloader.trouble(u'ERROR: unable to extract video url')
3586                         return
3587                 video_url = urllib.unquote(result.group('url').decode('utf-8'))
3588                 self.report_extract_entry(video_url)
3589
3590                 #Get the uploaded date
3591                 result = re.search(self.VIDEO_UPLOADED_RE, webpage)
3592                 if result is None:
3593                         self._downloader.trouble(u'ERROR: unable to extract video title')
3594                         return
3595                 upload_date = result.group('date').decode('utf-8')
3596                 self.report_date(upload_date);
3597
3598
3599                 info = {'id': video_id,
3600                                 'url': video_url,
3601                                 'uploader': None,
3602                                 'upload_date': upload_date,
3603                                 'title': video_title,
3604                                 'ext': 'flv',
3605                                 'format': 'flv',
3606                                 'thumbnail': None,
3607                                 'description': None,
3608                                 'player_url': None}
3609
3610                 return [info]
3611
3612
3613
3614
3615 class YouJizzIE(InfoExtractor):
3616         """Information extractor for youjizz.com."""
3617
3618         _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/([^.]+).html$'
3619         IE_NAME = u'youjizz'
3620         VIDEO_TITLE_RE = r'<title>(?P<title>.*)</title>'
3621         EMBED_PAGE_RE = r'http://www.youjizz.com/videos/embed/(?P<videoid>[0-9]+)'
3622         SOURCE_RE = r'so.addVariable\("file",encodeURIComponent\("(?P<source>[^"]+)"\)\);'
3623
3624         def __init__(self, downloader=None):
3625                 InfoExtractor.__init__(self, downloader)
3626
3627         def report_extract_entry(self, url):
3628                 """Report downloading extry"""
3629                 self._downloader.to_screen(u'[youjizz] Downloading entry: %s' % url.decode('utf-8'))
3630
3631         def report_webpage(self, url):
3632                 """Report downloading page"""
3633                 self._downloader.to_screen(u'[youjizz] Downloaded page: %s' % url)
3634
3635         def report_title(self, video_title):
3636                 """Report downloading extry"""
3637                 self._downloader.to_screen(u'[youjizz] Title: %s' % video_title.decode('utf-8'))
3638
3639         def report_embed_page(self, embed_page):
3640                 """Report downloading extry"""
3641                 self._downloader.to_screen(u'[youjizz] Embed Page: %s' % embed_page.decode('utf-8'))
3642
3643         def _real_extract(self, url):
3644                 # Get webpage content
3645                 try:
3646                         webpage = urllib2.urlopen(url).read()
3647                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3648                         self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3649                         return
3650                 self.report_webpage(url)
3651
3652                 # Get the video title
3653                 result = re.search(self.VIDEO_TITLE_RE, webpage)
3654                 if result is None:
3655                         self._downloader.trouble(u'ERROR: unable to extract video title')
3656                         return
3657                 video_title = result.group('title').decode('utf-8').strip()
3658                 self.report_title(video_title)
3659
3660                 # Get the embed page
3661                 result = re.search(self.EMBED_PAGE_RE, webpage)
3662                 if result is None:
3663                         self._downloader.trouble(u'ERROR: unable to extract embed page')
3664                         return
3665
3666                 embed_page_url = result.group(0).decode('utf-8').strip()
3667                 video_id = result.group('videoid').decode('utf-8')
3668                 self.report_embed_page(embed_page_url)
3669
3670                 try:
3671                         webpage = urllib2.urlopen(embed_page_url).read()
3672                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3673                         self._downloader.trouble(u'ERROR: unable to download video embed page: %s' % err)
3674                         return
3675
3676                 # Get the video URL
3677                 result = re.search(self.SOURCE_RE, webpage)
3678                 if result is None:
3679                         self._downloader.trouble(u'ERROR: unable to extract video url')
3680                         return
3681                 video_url = result.group('source').decode('utf-8')
3682                 self.report_extract_entry(video_url)
3683
3684                 info = {'id': video_id,
3685                                 'url': video_url,
3686                                 'uploader': None,
3687                                 'upload_date': None,
3688                                 'title': video_title,
3689                                 'ext': 'flv',
3690                                 'format': 'flv',
3691                                 'thumbnail': None,
3692                                 'description': None,
3693                                 'player_url': embed_page_url}
3694
3695                 return [info]
3696