_ Git - youtube-dl/blob - youtube_dl/InfoExtractors.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import datetime
   5 import netrc
   6 import os
   7 import re
   8 import socket
   9 import time
  10 import email.utils
  11 import xml.etree.ElementTree
  12 import random
  13 import math
  14
  15 from utils import *
  16
  17
  18 class InfoExtractor(object):
  19     """Information Extractor class.
  20
  21     Information extractors are the classes that, given a URL, extract
  22     information about the video (or videos) the URL refers to. This
  23     information includes the real video URL, the video title, author and
  24     others. The information is stored in a dictionary which is then
  25     passed to the FileDownloader. The FileDownloader processes this
  26     information possibly downloading the video to the file system, among
  27     other possible outcomes.
  28
  29     The dictionaries must include the following fields:
  30
  31     id:             Video identifier.
  32     url:            Final video URL.
  33     uploader:       Nickname of the video uploader, unescaped.
  34     upload_date:    Video upload date (YYYYMMDD).
  35     title:          Video title, unescaped.
  36     ext:            Video filename extension.
  37
  38     The following fields are optional:
  39
  40     format:         The video format, defaults to ext (used for --get-format)
  41     thumbnail:      Full URL to a video thumbnail image.
  42     description:    One-line video description.
  43     player_url:     SWF Player URL (used for rtmpdump).
  44     subtitles:      The .srt file contents.
  45     urlhandle:      [internal] The urlHandle to be used to download the file,
  46                     like returned by urllib.request.urlopen
  47
  48     The fields should all be Unicode strings.
  49
  50     Subclasses of this one should re-define the _real_initialize() and
  51     _real_extract() methods and define a _VALID_URL regexp.
  52     Probably, they should also be added to the list of extractors.
  53
  54     _real_extract() must return a *list* of information dictionaries as
  55     described above.
  56
  57     Finally, the _WORKING attribute should be set to False for broken IEs
  58     in order to warn the users and skip the tests.
  59     """
  60
  61     _ready = False
  62     _downloader = None
  63     _WORKING = True
  64
  65     def __init__(self, downloader=None):
  66         """Constructor. Receives an optional downloader."""
  67         self._ready = False
  68         self.set_downloader(downloader)
  69
  70     def suitable(self, url):
  71         """Receives a URL and returns True if suitable for this IE."""
  72         return re.match(self._VALID_URL, url) is not None
  73
  74     def working(self):
  75         """Getter method for _WORKING."""
  76         return self._WORKING
  77
  78     def initialize(self):
  79         """Initializes an instance (authentication, etc)."""
  80         if not self._ready:
  81             self._real_initialize()
  82             self._ready = True
  83
  84     def extract(self, url):
  85         """Extracts URL information and returns it in list of dicts."""
  86         self.initialize()
  87         return self._real_extract(url)
  88
  89     def set_downloader(self, downloader):
  90         """Sets the downloader for this IE."""
  91         self._downloader = downloader
  92
  93     def _real_initialize(self):
  94         """Real initialization process. Redefine in subclasses."""
  95         pass
  96
  97     def _real_extract(self, url):
  98         """Real extraction process. Redefine in subclasses."""
  99         pass
 100
 101
 102 class YoutubeIE(InfoExtractor):
 103     """Information extractor for youtube.com."""
 104
 105     _VALID_URL = r"""^
 106                      (
 107                          (?:https?://)?                                       # http(s):// (optional)
 108                          (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
 109                             tube\.majestyc\.net/)                             # the various hostnames, with wildcard subdomains
 110                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 111                          (?!view_play_list|my_playlists|artist|playlist)      # ignore playlist URLs
 112                          (?:                                                  # the various things that can precede the ID:
 113                              (?:(?:v|embed|e)/)                               # v/ or embed/ or e/
 114                              |(?:                                             # or the v= param in all its forms
 115                                  (?:watch(?:_popup)?(?:\.php)?)?              # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 116                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 117                                  (?:.+&)?                                     # any other preceding param (like /?s=tuff&v=xxxx)
 118                                  v=
 119                              )
 120                          )?                                                   # optional -> youtube.com/xxxx is OK
 121                      )?                                                       # all until now is optional -> you can pass the naked ID
 122                      ([0-9A-Za-z_-]+)                                         # here is it! the YouTube video ID
 123                      (?(1).+)?                                                # if we found the ID, everything can follow
 124                      $"""
 125     _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 126     _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
 127     _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 128     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 129     _NETRC_MACHINE = 'youtube'
 130     # Listed in order of quality
 131     _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
 132     _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
 133     _video_extensions = {
 134         '13': '3gp',
 135         '17': 'mp4',
 136         '18': 'mp4',
 137         '22': 'mp4',
 138         '37': 'mp4',
 139         '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
 140         '43': 'webm',
 141         '44': 'webm',
 142         '45': 'webm',
 143         '46': 'webm',
 144     }
 145     _video_dimensions = {
 146         '5': '240x400',
 147         '6': '???',
 148         '13': '???',
 149         '17': '144x176',
 150         '18': '360x640',
 151         '22': '720x1280',
 152         '34': '360x640',
 153         '35': '480x854',
 154         '37': '1080x1920',
 155         '38': '3072x4096',
 156         '43': '360x640',
 157         '44': '480x854',
 158         '45': '720x1280',
 159         '46': '1080x1920',
 160     }
 161     IE_NAME = u'youtube'
 162
 163     def suitable(self, url):
 164         """Receives a URL and returns True if suitable for this IE."""
 165         return re.match(self._VALID_URL, url, re.VERBOSE) is not None
 166
 167     def report_lang(self):
 168         """Report attempt to set language."""
 169         self._downloader.to_screen(u'[youtube] Setting language')
 170
 171     def report_login(self):
 172         """Report attempt to log in."""
 173         self._downloader.to_screen(u'[youtube] Logging in')
 174
 175     def report_age_confirmation(self):
 176         """Report attempt to confirm age."""
 177         self._downloader.to_screen(u'[youtube] Confirming age')
 178
 179     def report_video_webpage_download(self, video_id):
 180         """Report attempt to download video webpage."""
 181         self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
 182
 183     def report_video_info_webpage_download(self, video_id):
 184         """Report attempt to download video info webpage."""
 185         self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
 186
 187     def report_video_subtitles_download(self, video_id):
 188         """Report attempt to download video info webpage."""
 189         self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
 190
 191     def report_information_extraction(self, video_id):
 192         """Report attempt to extract video information."""
 193         self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
 194
 195     def report_unavailable_format(self, video_id, format):
 196         """Report extracted video URL."""
 197         self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
 198
 199     def report_rtmp_download(self):
 200         """Indicate the download will use the RTMP protocol."""
 201         self._downloader.to_screen(u'[youtube] RTMP download detected')
 202
 203     def _closed_captions_xml_to_srt(self, xml_string):
 204         srt = ''
 205         texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
 206         # TODO parse xml instead of regex
 207         for n, (start, dur_tag, dur, caption) in enumerate(texts):
 208             if not dur: dur = '4'
 209             start = float(start)
 210             end = start + float(dur)
 211             start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
 212             end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
 213             caption = unescapeHTML(caption)
 214             caption = unescapeHTML(caption) # double cycle, intentional
 215             srt += str(n+1) + '\n'
 216             srt += start + ' --> ' + end + '\n'
 217             srt += caption + '\n\n'
 218         return srt
 219
 220     def _print_formats(self, formats):
 221         print('Available formats:')
 222         for x in formats:
 223             print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
 224
 225     def _real_initialize(self):
 226         if self._downloader is None:
 227             return
 228
 229         username = None
 230         password = None
 231         downloader_params = self._downloader.params
 232
 233         # Attempt to use provided username and password or .netrc data
 234         if downloader_params.get('username', None) is not None:
 235             username = downloader_params['username']
 236             password = downloader_params['password']
 237         elif downloader_params.get('usenetrc', False):
 238             try:
 239                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 240                 if info is not None:
 241                     username = info[0]
 242                     password = info[2]
 243                 else:
 244                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 245             except (IOError, netrc.NetrcParseError) as err:
 246                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
 247                 return
 248
 249         # Set language
 250         request = compat_urllib_request.Request(self._LANG_URL)
 251         try:
 252             self.report_lang()
 253             compat_urllib_request.urlopen(request).read()
 254         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 255             self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
 256             return
 257
 258         # No authentication to be performed
 259         if username is None:
 260             return
 261
 262         # Log in
 263         login_form = {
 264                 'current_form': 'loginForm',
 265                 'next':     '/',
 266                 'action_login': 'Log In',
 267                 'username': username,
 268                 'password': password,
 269                 }
 270         request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
 271         try:
 272             self.report_login()
 273             login_results = compat_urllib_request.urlopen(request).read()
 274             if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 275                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 276                 return
 277         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 278             self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
 279             return
 280
 281         # Confirm age
 282         age_form = {
 283                 'next_url':     '/',
 284                 'action_confirm':   'Confirm',
 285                 }
 286         request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
 287         try:
 288             self.report_age_confirmation()
 289             age_results = compat_urllib_request.urlopen(request).read()
 290         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 291             self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
 292             return
 293
 294     def _real_extract(self, url):
 295         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
 296         mobj = re.search(self._NEXT_URL_RE, url)
 297         if mobj:
 298             url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
 299
 300         # Extract video id from URL
 301         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
 302         if mobj is None:
 303             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 304             return
 305         video_id = mobj.group(2)
 306
 307         # Get video webpage
 308         self.report_video_webpage_download(video_id)
 309         request = compat_urllib_request.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
 310         try:
 311             video_webpage_bytes = compat_urllib_request.urlopen(request).read()
 312         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 313             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 314             return
 315
 316         video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
 317
 318         # Attempt to extract SWF player URL
 319         mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
 320         if mobj is not None:
 321             player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
 322         else:
 323             player_url = None
 324
 325         # Get video info
 326         self.report_video_info_webpage_download(video_id)
 327         for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
 328             video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
 329                     % (video_id, el_type))
 330             request = compat_urllib_request.Request(video_info_url)
 331             try:
 332                 video_info_webpage_bytes = compat_urllib_request.urlopen(request).read()
 333                 video_info_webpage = video_info_webpage_bytes.decode('utf-8', 'ignore')
 334                 video_info = compat_parse_qs(video_info_webpage)
 335                 if 'token' in video_info:
 336                     break
 337             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 338                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
 339                 return
 340         if 'token' not in video_info:
 341             if 'reason' in video_info:
 342                 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0])
 343             else:
 344                 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
 345             return
 346
 347         # Check for "rental" videos
 348         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
 349             self._downloader.trouble(u'ERROR: "rental" videos not supported')
 350             return
 351
 352         # Start extracting information
 353         self.report_information_extraction(video_id)
 354
 355         # uploader
 356         if 'author' not in video_info:
 357             self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 358             return
 359         video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
 360
 361         # title
 362         if 'title' not in video_info:
 363             self._downloader.trouble(u'ERROR: unable to extract video title')
 364             return
 365         video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
 366
 367         # thumbnail image
 368         if 'thumbnail_url' not in video_info:
 369             self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
 370             video_thumbnail = ''
 371         else:   # don't panic if we can't find it
 372             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
 373
 374         # upload date
 375         upload_date = None
 376         mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
 377         if mobj is not None:
 378             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
 379             format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
 380             for expression in format_expressions:
 381                 try:
 382                     upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
 383                 except:
 384                     pass
 385
 386         # description
 387         video_description = get_element_by_id("eow-description", video_webpage)
 388         if video_description:
 389             video_description = clean_html(video_description)
 390         else:
 391             video_description = ''
 392
 393         # closed captions
 394         video_subtitles = None
 395         if self._downloader.params.get('writesubtitles', False):
 396             try:
 397                 self.report_video_subtitles_download(video_id)
 398                 request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
 399                 try:
 400                     srt_list = compat_urllib_request.urlopen(request).read()
 401                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 402                     raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
 403                 srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
 404                 srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
 405                 if not srt_lang_list:
 406                     raise Trouble(u'WARNING: video has no closed captions')
 407                 if self._downloader.params.get('subtitleslang', False):
 408                     srt_lang = self._downloader.params.get('subtitleslang')
 409                 elif 'en' in srt_lang_list:
 410                     srt_lang = 'en'
 411                 else:
 412                     srt_lang = srt_lang_list.keys()[0]
 413                 if not srt_lang in srt_lang_list:
 414                     raise Trouble(u'WARNING: no closed captions found in the specified language')
 415                 request = compat_urllib_request.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
 416                 try:
 417                     srt_xml = compat_urllib_request.urlopen(request).read()
 418                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 419                     raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
 420                 if not srt_xml:
 421                     raise Trouble(u'WARNING: unable to download video subtitles')
 422                 video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
 423             except Trouble as trouble:
 424                 self._downloader.trouble(trouble[0])
 425
 426         if 'length_seconds' not in video_info:
 427             self._downloader.trouble(u'WARNING: unable to extract video duration')
 428             video_duration = ''
 429         else:
 430             video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
 431
 432         # token
 433         video_token = compat_urllib_parse.unquote_plus(video_info['token'][0])
 434
 435         # Decide which formats to download
 436         req_format = self._downloader.params.get('format', None)
 437
 438         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
 439             self.report_rtmp_download()
 440             video_url_list = [(None, video_info['conn'][0])]
 441         elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
 442             url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
 443             url_data = [compat_parse_qs(uds) for uds in url_data_strs]
 444             url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
 445             url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
 446
 447             format_limit = self._downloader.params.get('format_limit', None)
 448             available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
 449             if format_limit is not None and format_limit in available_formats:
 450                 format_list = available_formats[available_formats.index(format_limit):]
 451             else:
 452                 format_list = available_formats
 453             existing_formats = [x for x in format_list if x in url_map]
 454             if len(existing_formats) == 0:
 455                 self._downloader.trouble(u'ERROR: no known formats available for video')
 456                 return
 457             if self._downloader.params.get('listformats', None):
 458                 self._print_formats(existing_formats)
 459                 return
 460             if req_format is None or req_format == 'best':
 461                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
 462             elif req_format == 'worst':
 463                 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
 464             elif req_format in ('-1', 'all'):
 465                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
 466             else:
 467                 # Specific formats. We pick the first in a slash-delimeted sequence.
 468                 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
 469                 req_formats = req_format.split('/')
 470                 video_url_list = None
 471                 for rf in req_formats:
 472                     if rf in url_map:
 473                         video_url_list = [(rf, url_map[rf])]
 474                         break
 475                 if video_url_list is None:
 476                     self._downloader.trouble(u'ERROR: requested format not available')
 477                     return
 478         else:
 479             self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
 480             return
 481
 482         results = []
 483         for format_param, video_real_url in video_url_list:
 484             # Extension
 485             video_extension = self._video_extensions.get(format_param, 'flv')
 486
 487             video_format = '{} - {}'.format(format_param if format_param else video_extension,
 488                                             self._video_dimensions.get(format_param, '???'))
 489
 490             results.append({
 491                 'id':       video_id,
 492                 'url':      video_real_url,
 493                 'uploader': video_uploader,
 494                 'upload_date':  upload_date,
 495                 'title':    video_title,
 496                 'ext':      video_extension,
 497                 'format':   video_format,
 498                 'thumbnail':    video_thumbnail,
 499                 'description':  video_description,
 500                 'player_url':   player_url,
 501                 'subtitles':    video_subtitles,
 502                 'duration':     video_duration
 503             })
 504         return results
 505
 506
 507 class MetacafeIE(InfoExtractor):
 508     """Information Extractor for metacafe.com."""
 509
 510     _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 511     _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 512     _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 513     IE_NAME = u'metacafe'
 514
 515     def __init__(self, downloader=None):
 516         InfoExtractor.__init__(self, downloader)
 517
 518     def report_disclaimer(self):
 519         """Report disclaimer retrieval."""
 520         self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
 521
 522     def report_age_confirmation(self):
 523         """Report attempt to confirm age."""
 524         self._downloader.to_screen(u'[metacafe] Confirming age')
 525
 526     def report_download_webpage(self, video_id):
 527         """Report webpage download."""
 528         self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
 529
 530     def report_extraction(self, video_id):
 531         """Report information extraction."""
 532         self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
 533
 534     def _real_initialize(self):
 535         # Retrieve disclaimer
 536         request = compat_urllib_request.Request(self._DISCLAIMER)
 537         try:
 538             self.report_disclaimer()
 539             disclaimer = compat_urllib_request.urlopen(request).read()
 540         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 541             self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
 542             return
 543
 544         # Confirm age
 545         disclaimer_form = {
 546             'filters': '0',
 547             'submit': "Continue - I'm over 18",
 548             }
 549         request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
 550         try:
 551             self.report_age_confirmation()
 552             disclaimer = compat_urllib_request.urlopen(request).read()
 553         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 554             self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
 555             return
 556
 557     def _real_extract(self, url):
 558         # Extract id and simplified title from URL
 559         mobj = re.match(self._VALID_URL, url)
 560         if mobj is None:
 561             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 562             return
 563
 564         video_id = mobj.group(1)
 565
 566         # Check if video comes from YouTube
 567         mobj2 = re.match(r'^yt-(.*)$', video_id)
 568         if mobj2 is not None:
 569             self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
 570             return
 571
 572         # Retrieve video webpage to extract further information
 573         request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
 574         try:
 575             self.report_download_webpage(video_id)
 576             webpage = compat_urllib_request.urlopen(request).read()
 577         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 578             self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
 579             return
 580
 581         # Extract URL, uploader and title from webpage
 582         self.report_extraction(video_id)
 583         mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 584         if mobj is not None:
 585             mediaURL = compat_urllib_parse.unquote(mobj.group(1))
 586             video_extension = mediaURL[-3:]
 587
 588             # Extract gdaKey if available
 589             mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 590             if mobj is None:
 591                 video_url = mediaURL
 592             else:
 593                 gdaKey = mobj.group(1)
 594                 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 595         else:
 596             mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
 597             if mobj is None:
 598                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 599                 return
 600             vardict = compat_parse_qs(mobj.group(1))
 601             if 'mediaData' not in vardict:
 602                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 603                 return
 604             mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
 605             if mobj is None:
 606                 self._downloader.trouble(u'ERROR: unable to extract media URL')
 607                 return
 608             mediaURL = mobj.group(1).replace('\\/', '/')
 609             video_extension = mediaURL[-3:]
 610             video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
 611
 612         mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 613         if mobj is None:
 614             self._downloader.trouble(u'ERROR: unable to extract title')
 615             return
 616         video_title = mobj.group(1).decode('utf-8')
 617
 618         mobj = re.search(r'submitter=(.*?);', webpage)
 619         if mobj is None:
 620             self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 621             return
 622         video_uploader = mobj.group(1)
 623
 624         return [{
 625             'id':       video_id.decode('utf-8'),
 626             'url':      video_url.decode('utf-8'),
 627             'uploader': video_uploader.decode('utf-8'),
 628             'upload_date':  None,
 629             'title':    video_title,
 630             'ext':      video_extension.decode('utf-8'),
 631         }]
 632
 633
 634 class DailymotionIE(InfoExtractor):
 635     """Information Extractor for Dailymotion"""
 636
 637     _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
 638     IE_NAME = u'dailymotion'
 639
 640     def __init__(self, downloader=None):
 641         InfoExtractor.__init__(self, downloader)
 642
 643     def report_download_webpage(self, video_id):
 644         """Report webpage download."""
 645         self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
 646
 647     def report_extraction(self, video_id):
 648         """Report information extraction."""
 649         self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
 650
 651     def _real_extract(self, url):
 652         # Extract id and simplified title from URL
 653         mobj = re.match(self._VALID_URL, url)
 654         if mobj is None:
 655             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 656             return
 657
 658         video_id = mobj.group(1).split('_')[0].split('?')[0]
 659
 660         video_extension = 'mp4'
 661
 662         # Retrieve video webpage to extract further information
 663         request = compat_urllib_request.Request(url)
 664         request.add_header('Cookie', 'family_filter=off')
 665         try:
 666             self.report_download_webpage(video_id)
 667             webpage = compat_urllib_request.urlopen(request).read()
 668         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 669             self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
 670             return
 671
 672         # Extract URL, uploader and title from webpage
 673         self.report_extraction(video_id)
 674         mobj = re.search(r'\s*var flashvars = (.*)', webpage)
 675         if mobj is None:
 676             self._downloader.trouble(u'ERROR: unable to extract media URL')
 677             return
 678         flashvars = compat_urllib_parse.unquote(mobj.group(1))
 679
 680         for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
 681             if key in flashvars:
 682                 max_quality = key
 683                 self._downloader.to_screen(u'[dailymotion] Using %s' % key)
 684                 break
 685         else:
 686             self._downloader.trouble(u'ERROR: unable to extract video URL')
 687             return
 688
 689         mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
 690         if mobj is None:
 691             self._downloader.trouble(u'ERROR: unable to extract video URL')
 692             return
 693
 694         video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/')
 695
 696         # TODO: support choosing qualities
 697
 698         mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
 699         if mobj is None:
 700             self._downloader.trouble(u'ERROR: unable to extract title')
 701             return
 702         video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
 703
 704         video_uploader = None
 705         mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
 706         if mobj is None:
 707             # lookin for official user
 708             mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
 709             if mobj_official is None:
 710                 self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
 711             else:
 712                 video_uploader = mobj_official.group(1)
 713         else:
 714             video_uploader = mobj.group(1)
 715
 716         video_upload_date = None
 717         mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
 718         if mobj is not None:
 719             video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
 720
 721         return [{
 722             'id':       video_id.decode('utf-8'),
 723             'url':      video_url.decode('utf-8'),
 724             'uploader': video_uploader.decode('utf-8'),
 725             'upload_date':  video_upload_date,
 726             'title':    video_title,
 727             'ext':      video_extension.decode('utf-8'),
 728         }]
 729
 730
 731 class GoogleIE(InfoExtractor):
 732     """Information extractor for video.google.com."""
 733
 734     _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
 735     IE_NAME = u'video.google'
 736
 737     def __init__(self, downloader=None):
 738         InfoExtractor.__init__(self, downloader)
 739
 740     def report_download_webpage(self, video_id):
 741         """Report webpage download."""
 742         self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
 743
 744     def report_extraction(self, video_id):
 745         """Report information extraction."""
 746         self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
 747
 748     def _real_extract(self, url):
 749         # Extract id from URL
 750         mobj = re.match(self._VALID_URL, url)
 751         if mobj is None:
 752             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 753             return
 754
 755         video_id = mobj.group(1)
 756
 757         video_extension = 'mp4'
 758
 759         # Retrieve video webpage to extract further information
 760         request = compat_urllib_request.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
 761         try:
 762             self.report_download_webpage(video_id)
 763             webpage = compat_urllib_request.urlopen(request).read()
 764         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 765             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 766             return
 767
 768         # Extract URL, uploader, and title from webpage
 769         self.report_extraction(video_id)
 770         mobj = re.search(r"download_url:'([^']+)'", webpage)
 771         if mobj is None:
 772             video_extension = 'flv'
 773             mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
 774         if mobj is None:
 775             self._downloader.trouble(u'ERROR: unable to extract media URL')
 776             return
 777         mediaURL = compat_urllib_parse.unquote(mobj.group(1))
 778         mediaURL = mediaURL.replace('\\x3d', '\x3d')
 779         mediaURL = mediaURL.replace('\\x26', '\x26')
 780
 781         video_url = mediaURL
 782
 783         mobj = re.search(r'<title>(.*)</title>', webpage)
 784         if mobj is None:
 785             self._downloader.trouble(u'ERROR: unable to extract title')
 786             return
 787         video_title = mobj.group(1).decode('utf-8')
 788
 789         # Extract video description
 790         mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
 791         if mobj is None:
 792             self._downloader.trouble(u'ERROR: unable to extract video description')
 793             return
 794         video_description = mobj.group(1).decode('utf-8')
 795         if not video_description:
 796             video_description = 'No description available.'
 797
 798         # Extract video thumbnail
 799         if self._downloader.params.get('forcethumbnail', False):
 800             request = compat_urllib_request.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
 801             try:
 802                 webpage = compat_urllib_request.urlopen(request).read()
 803             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 804                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 805                 return
 806             mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
 807             if mobj is None:
 808                 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
 809                 return
 810             video_thumbnail = mobj.group(1)
 811         else:   # we need something to pass to process_info
 812             video_thumbnail = ''
 813
 814         return [{
 815             'id':       video_id.decode('utf-8'),
 816             'url':      video_url.decode('utf-8'),
 817             'uploader': None,
 818             'upload_date':  None,
 819             'title':    video_title,
 820             'ext':      video_extension.decode('utf-8'),
 821         }]
 822
 823
 824 class PhotobucketIE(InfoExtractor):
 825     """Information extractor for photobucket.com."""
 826
 827     _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
 828     IE_NAME = u'photobucket'
 829
 830     def __init__(self, downloader=None):
 831         InfoExtractor.__init__(self, downloader)
 832
 833     def report_download_webpage(self, video_id):
 834         """Report webpage download."""
 835         self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
 836
 837     def report_extraction(self, video_id):
 838         """Report information extraction."""
 839         self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
 840
 841     def _real_extract(self, url):
 842         # Extract id from URL
 843         mobj = re.match(self._VALID_URL, url)
 844         if mobj is None:
 845             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 846             return
 847
 848         video_id = mobj.group(1)
 849
 850         video_extension = 'flv'
 851
 852         # Retrieve video webpage to extract further information
 853         request = compat_urllib_request.Request(url)
 854         try:
 855             self.report_download_webpage(video_id)
 856             webpage = compat_urllib_request.urlopen(request).read()
 857         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 858             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 859             return
 860
 861         # Extract URL, uploader, and title from webpage
 862         self.report_extraction(video_id)
 863         mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
 864         if mobj is None:
 865             self._downloader.trouble(u'ERROR: unable to extract media URL')
 866             return
 867         mediaURL = compat_urllib_parse.unquote(mobj.group(1))
 868
 869         video_url = mediaURL
 870
 871         mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
 872         if mobj is None:
 873             self._downloader.trouble(u'ERROR: unable to extract title')
 874             return
 875         video_title = mobj.group(1).decode('utf-8')
 876
 877         video_uploader = mobj.group(2).decode('utf-8')
 878
 879         return [{
 880             'id':       video_id.decode('utf-8'),
 881             'url':      video_url.decode('utf-8'),
 882             'uploader': video_uploader,
 883             'upload_date':  None,
 884             'title':    video_title,
 885             'ext':      video_extension.decode('utf-8'),
 886         }]
 887
 888
 889 class YahooIE(InfoExtractor):
 890     """Information extractor for video.yahoo.com."""
 891
 892     # _VALID_URL matches all Yahoo! Video URLs
 893     # _VPAGE_URL matches only the extractable '/watch/' URLs
 894     _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
 895     _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
 896     IE_NAME = u'video.yahoo'
 897
 898     def __init__(self, downloader=None):
 899         InfoExtractor.__init__(self, downloader)
 900
 901     def report_download_webpage(self, video_id):
 902         """Report webpage download."""
 903         self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
 904
 905     def report_extraction(self, video_id):
 906         """Report information extraction."""
 907         self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
 908
 909     def _real_extract(self, url, new_video=True):
 910         # Extract ID from URL
 911         mobj = re.match(self._VALID_URL, url)
 912         if mobj is None:
 913             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
 914             return
 915
 916         video_id = mobj.group(2)
 917         video_extension = 'flv'
 918
 919         # Rewrite valid but non-extractable URLs as
 920         # extractable English language /watch/ URLs
 921         if re.match(self._VPAGE_URL, url) is None:
 922             request = compat_urllib_request.Request(url)
 923             try:
 924                 webpage = compat_urllib_request.urlopen(request).read()
 925             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 926                 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 927                 return
 928
 929             mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
 930             if mobj is None:
 931                 self._downloader.trouble(u'ERROR: Unable to extract id field')
 932                 return
 933             yahoo_id = mobj.group(1)
 934
 935             mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
 936             if mobj is None:
 937                 self._downloader.trouble(u'ERROR: Unable to extract vid field')
 938                 return
 939             yahoo_vid = mobj.group(1)
 940
 941             url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
 942             return self._real_extract(url, new_video=False)
 943
 944         # Retrieve video webpage to extract further information
 945         request = compat_urllib_request.Request(url)
 946         try:
 947             self.report_download_webpage(video_id)
 948             webpage = compat_urllib_request.urlopen(request).read()
 949         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 950             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 951             return
 952
 953         # Extract uploader and title from webpage
 954         self.report_extraction(video_id)
 955         mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
 956         if mobj is None:
 957             self._downloader.trouble(u'ERROR: unable to extract video title')
 958             return
 959         video_title = mobj.group(1).decode('utf-8')
 960
 961         mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
 962         if mobj is None:
 963             self._downloader.trouble(u'ERROR: unable to extract video uploader')
 964             return
 965         video_uploader = mobj.group(1).decode('utf-8')
 966
 967         # Extract video thumbnail
 968         mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
 969         if mobj is None:
 970             self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
 971             return
 972         video_thumbnail = mobj.group(1).decode('utf-8')
 973
 974         # Extract video description
 975         mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
 976         if mobj is None:
 977             self._downloader.trouble(u'ERROR: unable to extract video description')
 978             return
 979         video_description = mobj.group(1).decode('utf-8')
 980         if not video_description:
 981             video_description = 'No description available.'
 982
 983         # Extract video height and width
 984         mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
 985         if mobj is None:
 986             self._downloader.trouble(u'ERROR: unable to extract video height')
 987             return
 988         yv_video_height = mobj.group(1)
 989
 990         mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
 991         if mobj is None:
 992             self._downloader.trouble(u'ERROR: unable to extract video width')
 993             return
 994         yv_video_width = mobj.group(1)
 995
 996         # Retrieve video playlist to extract media URL
 997         # I'm not completely sure what all these options are, but we
 998         # seem to need most of them, otherwise the server sends a 401.
 999         yv_lg = 'R0xx6idZnW2zlrKP8xxAIR'  # not sure what this represents
1000         yv_bitrate = '700'  # according to Wikipedia this is hard-coded
1001         request = compat_urllib_request.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1002                 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1003                 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1004         try:
1005             self.report_download_webpage(video_id)
1006             webpage = compat_urllib_request.urlopen(request).read()
1007         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1008             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1009             return
1010
1011         # Extract media URL from playlist XML
1012         mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1013         if mobj is None:
1014             self._downloader.trouble(u'ERROR: Unable to extract media URL')
1015             return
1016         video_url = compat_urllib_parse.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1017         video_url = unescapeHTML(video_url)
1018
1019         return [{
1020             'id':       video_id.decode('utf-8'),
1021             'url':      video_url,
1022             'uploader': video_uploader,
1023             'upload_date':  None,
1024             'title':    video_title,
1025             'ext':      video_extension.decode('utf-8'),
1026             'thumbnail':    video_thumbnail.decode('utf-8'),
1027             'description':  video_description,
1028         }]
1029
1030
1031 class VimeoIE(InfoExtractor):
1032     """Information extractor for vimeo.com."""
1033
1034     # _VALID_URL matches Vimeo URLs
1035     _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?:videos?/)?([0-9]+)'
1036     IE_NAME = u'vimeo'
1037
1038     def __init__(self, downloader=None):
1039         InfoExtractor.__init__(self, downloader)
1040
1041     def report_download_webpage(self, video_id):
1042         """Report webpage download."""
1043         self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1044
1045     def report_extraction(self, video_id):
1046         """Report information extraction."""
1047         self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1048
1049     def _real_extract(self, url, new_video=True):
1050         # Extract ID from URL
1051         mobj = re.match(self._VALID_URL, url)
1052         if mobj is None:
1053             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1054             return
1055
1056         video_id = mobj.group(1)
1057
1058         # Retrieve video webpage to extract further information
1059         request = compat_urllib_request.Request(url, None, std_headers)
1060         try:
1061             self.report_download_webpage(video_id)
1062             webpage = compat_urllib_request.urlopen(request).read()
1063         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1064             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1065             return
1066
1067         # Now we begin extracting as much information as we can from what we
1068         # retrieved. First we extract the information common to all extractors,
1069         # and latter we extract those that are Vimeo specific.
1070         self.report_extraction(video_id)
1071
1072         # Extract the config JSON
1073         config = webpage.split(' = {config:')[1].split(',assets:')[0]
1074         try:
1075             config = json.loads(config)
1076         except:
1077             self._downloader.trouble(u'ERROR: unable to extract info section')
1078             return
1079
1080         # Extract title
1081         video_title = config["video"]["title"]
1082
1083         # Extract uploader
1084         video_uploader = config["video"]["owner"]["name"]
1085
1086         # Extract video thumbnail
1087         video_thumbnail = config["video"]["thumbnail"]
1088
1089         # Extract video description
1090         video_description = get_element_by_id("description", webpage.decode('utf8'))
1091         if video_description: video_description = clean_html(video_description)
1092         else: video_description = ''
1093
1094         # Extract upload date
1095         video_upload_date = None
1096         mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
1097         if mobj is not None:
1098             video_upload_date = mobj.group(1)
1099
1100         # Vimeo specific: extract request signature and timestamp
1101         sig = config['request']['signature']
1102         timestamp = config['request']['timestamp']
1103
1104         # Vimeo specific: extract video codec and quality information
1105         # First consider quality, then codecs, then take everything
1106         # TODO bind to format param
1107         codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
1108         files = { 'hd': [], 'sd': [], 'other': []}
1109         for codec_name, codec_extension in codecs:
1110             if codec_name in config["video"]["files"]:
1111                 if 'hd' in config["video"]["files"][codec_name]:
1112                     files['hd'].append((codec_name, codec_extension, 'hd'))
1113                 elif 'sd' in config["video"]["files"][codec_name]:
1114                     files['sd'].append((codec_name, codec_extension, 'sd'))
1115                 else:
1116                     files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0]))
1117
1118         for quality in ('hd', 'sd', 'other'):
1119             if len(files[quality]) > 0:
1120                 video_quality = files[quality][0][2]
1121                 video_codec = files[quality][0][0]
1122                 video_extension = files[quality][0][1]
1123                 self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
1124                 break
1125         else:
1126             self._downloader.trouble(u'ERROR: no known codec found')
1127             return
1128
1129         video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
1130                     %(video_id, sig, timestamp, video_quality, video_codec.upper())
1131
1132         return [{
1133             'id':       video_id,
1134             'url':      video_url,
1135             'uploader': video_uploader,
1136             'upload_date':  video_upload_date,
1137             'title':    video_title,
1138             'ext':      video_extension,
1139             'thumbnail':    video_thumbnail,
1140             'description':  video_description,
1141         }]
1142
1143
1144 class ArteTvIE(InfoExtractor):
1145     """arte.tv information extractor."""
1146
1147     _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
1148     _LIVE_URL = r'index-[0-9]+\.html$'
1149
1150     IE_NAME = u'arte.tv'
1151
1152     def __init__(self, downloader=None):
1153         InfoExtractor.__init__(self, downloader)
1154
1155     def report_download_webpage(self, video_id):
1156         """Report webpage download."""
1157         self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
1158
1159     def report_extraction(self, video_id):
1160         """Report information extraction."""
1161         self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
1162
1163     def fetch_webpage(self, url):
1164         self._downloader.increment_downloads()
1165         request = compat_urllib_request.Request(url)
1166         try:
1167             self.report_download_webpage(url)
1168             webpage = compat_urllib_request.urlopen(request).read()
1169         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1170             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1171             return
1172         except ValueError as err:
1173             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1174             return
1175         return webpage
1176
1177     def grep_webpage(self, url, regex, regexFlags, matchTuples):
1178         page = self.fetch_webpage(url)
1179         mobj = re.search(regex, page, regexFlags)
1180         info = {}
1181
1182         if mobj is None:
1183             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1184             return
1185
1186         for (i, key, err) in matchTuples:
1187             if mobj.group(i) is None:
1188                 self._downloader.trouble(err)
1189                 return
1190             else:
1191                 info[key] = mobj.group(i)
1192
1193         return info
1194
1195     def extractLiveStream(self, url):
1196         video_lang = url.split('/')[-4]
1197         info = self.grep_webpage(
1198             url,
1199             r'src="(.*?/videothek_js.*?\.js)',
1200             0,
1201             [
1202                 (1, 'url', u'ERROR: Invalid URL: %s' % url)
1203             ]
1204         )
1205         http_host = url.split('/')[2]
1206         next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url')))
1207         info = self.grep_webpage(
1208             next_url,
1209             r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
1210                 '(http://.*?\.swf).*?' +
1211                 '(rtmp://.*?)\'',
1212             re.DOTALL,
1213             [
1214                 (1, 'path',   u'ERROR: could not extract video path: %s' % url),
1215                 (2, 'player', u'ERROR: could not extract video player: %s' % url),
1216                 (3, 'url',    u'ERROR: could not extract video url: %s' % url)
1217             ]
1218         )
1219         video_url = u'%s/%s' % (info.get('url'), info.get('path'))
1220
1221     def extractPlus7Stream(self, url):
1222         video_lang = url.split('/')[-3]
1223         info = self.grep_webpage(
1224             url,
1225             r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
1226             0,
1227             [
1228                 (1, 'url', u'ERROR: Invalid URL: %s' % url)
1229             ]
1230         )
1231         next_url = compat_urllib_parse.unquote(info.get('url'))
1232         info = self.grep_webpage(
1233             next_url,
1234             r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
1235             0,
1236             [
1237                 (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
1238             ]
1239         )
1240         next_url = compat_urllib_parse.unquote(info.get('url'))
1241
1242         info = self.grep_webpage(
1243             next_url,
1244             r'<video id="(.*?)".*?>.*?' +
1245                 '<name>(.*?)</name>.*?' +
1246                 '<dateVideo>(.*?)</dateVideo>.*?' +
1247                 '<url quality="hd">(.*?)</url>',
1248             re.DOTALL,
1249             [
1250                 (1, 'id',    u'ERROR: could not extract video id: %s' % url),
1251                 (2, 'title', u'ERROR: could not extract video title: %s' % url),
1252                 (3, 'date',  u'ERROR: could not extract video date: %s' % url),
1253                 (4, 'url',   u'ERROR: could not extract video url: %s' % url)
1254             ]
1255         )
1256
1257         return {
1258             'id':           info.get('id'),
1259             'url':          compat_urllib_parse.unquote(info.get('url')),
1260             'uploader':     u'arte.tv',
1261             'upload_date':  info.get('date'),
1262             'title':        info.get('title'),
1263             'ext':          u'mp4',
1264             'format':       u'NA',
1265             'player_url':   None,
1266         }
1267
1268     def _real_extract(self, url):
1269         video_id = url.split('/')[-1]
1270         self.report_extraction(video_id)
1271
1272         if re.search(self._LIVE_URL, video_id) is not None:
1273             self.extractLiveStream(url)
1274             return
1275         else:
1276             info = self.extractPlus7Stream(url)
1277
1278         return [info]
1279
1280
1281 class GenericIE(InfoExtractor):
1282     """Generic last-resort information extractor."""
1283
1284     _VALID_URL = r'.*'
1285     IE_NAME = u'generic'
1286
1287     def __init__(self, downloader=None):
1288         InfoExtractor.__init__(self, downloader)
1289
1290     def report_download_webpage(self, video_id):
1291         """Report webpage download."""
1292         self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1293         self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1294
1295     def report_extraction(self, video_id):
1296         """Report information extraction."""
1297         self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1298
1299     def report_following_redirect(self, new_url):
1300         """Report information extraction."""
1301         self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
1302
1303     def _test_redirect(self, url):
1304         """Check if it is a redirect, like url shorteners, in case restart chain."""
1305         class HeadRequest(compat_urllib_request.Request):
1306             def get_method(self):
1307                 return "HEAD"
1308
1309         class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
1310             """
1311             Subclass the HTTPRedirectHandler to make it use our
1312             HeadRequest also on the redirected URL
1313             """
1314             def redirect_request(self, req, fp, code, msg, headers, newurl):
1315                 if code in (301, 302, 303, 307):
1316                     newurl = newurl.replace(' ', '%20')
1317                     newheaders = dict((k,v) for k,v in req.headers.items()
1318                                       if k.lower() not in ("content-length", "content-type"))
1319                     return HeadRequest(newurl,
1320                                        headers=newheaders,
1321                                        origin_req_host=req.get_origin_req_host(),
1322                                        unverifiable=True)
1323                 else:
1324                     raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
1325
1326         class HTTPMethodFallback(compat_urllib_request.BaseHandler):
1327             """
1328             Fallback to GET if HEAD is not allowed (405 HTTP error)
1329             """
1330             def http_error_405(self, req, fp, code, msg, headers):
1331                 fp.read()
1332                 fp.close()
1333
1334                 newheaders = dict((k,v) for k,v in req.headers.items()
1335                                   if k.lower() not in ("content-length", "content-type"))
1336                 return self.parent.open(compat_urllib_request.Request(req.get_full_url(),
1337                                                  headers=newheaders,
1338                                                  origin_req_host=req.get_origin_req_host(),
1339                                                  unverifiable=True))
1340
1341         # Build our opener
1342         opener = compat_urllib_request.OpenerDirector()
1343         for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
1344                         HTTPMethodFallback, HEADRedirectHandler,
1345                         compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
1346             opener.add_handler(handler())
1347
1348         response = opener.open(HeadRequest(url))
1349         new_url = response.geturl()
1350
1351         if url == new_url:
1352             return False
1353
1354         self.report_following_redirect(new_url)
1355         self._downloader.download([new_url])
1356         return True
1357
1358     def _real_extract(self, url):
1359         if self._test_redirect(url): return
1360
1361         video_id = url.split('/')[-1]
1362         request = compat_urllib_request.Request(url)
1363         try:
1364             self.report_download_webpage(video_id)
1365             webpage = compat_urllib_request.urlopen(request).read()
1366         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1367             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1368             return
1369         except ValueError as err:
1370             # since this is the last-resort InfoExtractor, if
1371             # this error is thrown, it'll be thrown here
1372             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1373             return
1374
1375         self.report_extraction(video_id)
1376         # Start with something easy: JW Player in SWFObject
1377         mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1378         if mobj is None:
1379             # Broaden the search a little bit
1380             mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1381         if mobj is None:
1382             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1383             return
1384
1385         # It's possible that one of the regexes
1386         # matched, but returned an empty group:
1387         if mobj.group(1) is None:
1388             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1389             return
1390
1391         video_url = compat_urllib_parse.unquote(mobj.group(1))
1392         video_id = os.path.basename(video_url)
1393
1394         # here's a fun little line of code for you:
1395         video_extension = os.path.splitext(video_id)[1][1:]
1396         video_id = os.path.splitext(video_id)[0]
1397
1398         # it's tempting to parse this further, but you would
1399         # have to take into account all the variations like
1400         #   Video Title - Site Name
1401         #   Site Name | Video Title
1402         #   Video Title - Tagline | Site Name
1403         # and so on and so forth; it's just not practical
1404         mobj = re.search(r'<title>(.*)</title>', webpage)
1405         if mobj is None:
1406             self._downloader.trouble(u'ERROR: unable to extract title')
1407             return
1408         video_title = mobj.group(1).decode('utf-8')
1409
1410         # video uploader is domain name
1411         mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1412         if mobj is None:
1413             self._downloader.trouble(u'ERROR: unable to extract title')
1414             return
1415         video_uploader = mobj.group(1).decode('utf-8')
1416
1417         return [{
1418             'id':       video_id.decode('utf-8'),
1419             'url':      video_url.decode('utf-8'),
1420             'uploader': video_uploader,
1421             'upload_date':  None,
1422             'title':    video_title,
1423             'ext':      video_extension.decode('utf-8'),
1424         }]
1425
1426
1427 class YoutubeSearchIE(InfoExtractor):
1428     """Information Extractor for YouTube search queries."""
1429     _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
1430     _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1431     _max_youtube_results = 1000
1432     IE_NAME = u'youtube:search'
1433
1434     def __init__(self, downloader=None):
1435         InfoExtractor.__init__(self, downloader)
1436
1437     def report_download_page(self, query, pagenum):
1438         """Report attempt to download search page with given number."""
1439         query = query.decode(preferredencoding())
1440         self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1441
1442     def _real_extract(self, query):
1443         mobj = re.match(self._VALID_URL, query)
1444         if mobj is None:
1445             self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1446             return
1447
1448         prefix, query = query.split(':')
1449         prefix = prefix[8:]
1450         query = query.encode('utf-8')
1451         if prefix == '':
1452             self._download_n_results(query, 1)
1453             return
1454         elif prefix == 'all':
1455             self._download_n_results(query, self._max_youtube_results)
1456             return
1457         else:
1458             try:
1459                 n = int(prefix)
1460                 if n <= 0:
1461                     self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1462                     return
1463                 elif n > self._max_youtube_results:
1464                     self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1465                     n = self._max_youtube_results
1466                 self._download_n_results(query, n)
1467                 return
1468             except ValueError: # parsing prefix as integer fails
1469                 self._download_n_results(query, 1)
1470                 return
1471
1472     def _download_n_results(self, query, n):
1473         """Downloads a specified number of results for a query"""
1474
1475         video_ids = []
1476         pagenum = 0
1477         limit = n
1478
1479         while (50 * pagenum) < limit:
1480             self.report_download_page(query, pagenum+1)
1481             result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
1482             request = compat_urllib_request.Request(result_url)
1483             try:
1484                 data = compat_urllib_request.urlopen(request).read()
1485             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1486                 self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
1487                 return
1488             api_response = json.loads(data)['data']
1489
1490             new_ids = list(video['id'] for video in api_response['items'])
1491             video_ids += new_ids
1492
1493             limit = min(n, api_response['totalItems'])
1494             pagenum += 1
1495
1496         if len(video_ids) > n:
1497             video_ids = video_ids[:n]
1498         for id in video_ids:
1499             self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1500         return
1501
1502
1503 class GoogleSearchIE(InfoExtractor):
1504     """Information Extractor for Google Video search queries."""
1505     _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
1506     _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1507     _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
1508     _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
1509     _max_google_results = 1000
1510     IE_NAME = u'video.google:search'
1511
1512     def __init__(self, downloader=None):
1513         InfoExtractor.__init__(self, downloader)
1514
1515     def report_download_page(self, query, pagenum):
1516         """Report attempt to download playlist page with given number."""
1517         query = query.decode(preferredencoding())
1518         self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1519
1520     def _real_extract(self, query):
1521         mobj = re.match(self._VALID_URL, query)
1522         if mobj is None:
1523             self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1524             return
1525
1526         prefix, query = query.split(':')
1527         prefix = prefix[8:]
1528         query = query.encode('utf-8')
1529         if prefix == '':
1530             self._download_n_results(query, 1)
1531             return
1532         elif prefix == 'all':
1533             self._download_n_results(query, self._max_google_results)
1534             return
1535         else:
1536             try:
1537                 n = int(prefix)
1538                 if n <= 0:
1539                     self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1540                     return
1541                 elif n > self._max_google_results:
1542                     self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1543                     n = self._max_google_results
1544                 self._download_n_results(query, n)
1545                 return
1546             except ValueError: # parsing prefix as integer fails
1547                 self._download_n_results(query, 1)
1548                 return
1549
1550     def _download_n_results(self, query, n):
1551         """Downloads a specified number of results for a query"""
1552
1553         video_ids = []
1554         pagenum = 0
1555
1556         while True:
1557             self.report_download_page(query, pagenum)
1558             result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum*10)
1559             request = compat_urllib_request.Request(result_url)
1560             try:
1561                 page = compat_urllib_request.urlopen(request).read()
1562             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1563                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1564                 return
1565
1566             # Extract video identifiers
1567             for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1568                 video_id = mobj.group(1)
1569                 if video_id not in video_ids:
1570                     video_ids.append(video_id)
1571                     if len(video_ids) == n:
1572                         # Specified n videos reached
1573                         for id in video_ids:
1574                             self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1575                         return
1576
1577             if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1578                 for id in video_ids:
1579                     self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1580                 return
1581
1582             pagenum = pagenum + 1
1583
1584
1585 class YahooSearchIE(InfoExtractor):
1586     """Information Extractor for Yahoo! Video search queries."""
1587     _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
1588     _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1589     _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1590     _MORE_PAGES_INDICATOR = r'\s*Next'
1591     _max_yahoo_results = 1000
1592     IE_NAME = u'video.yahoo:search'
1593
1594     def __init__(self, downloader=None):
1595         InfoExtractor.__init__(self, downloader)
1596
1597     def report_download_page(self, query, pagenum):
1598         """Report attempt to download playlist page with given number."""
1599         query = query.decode(preferredencoding())
1600         self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1601
1602     def _real_extract(self, query):
1603         mobj = re.match(self._VALID_URL, query)
1604         if mobj is None:
1605             self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1606             return
1607
1608         prefix, query = query.split(':')
1609         prefix = prefix[8:]
1610         query = query.encode('utf-8')
1611         if prefix == '':
1612             self._download_n_results(query, 1)
1613             return
1614         elif prefix == 'all':
1615             self._download_n_results(query, self._max_yahoo_results)
1616             return
1617         else:
1618             try:
1619                 n = int(prefix)
1620                 if n <= 0:
1621                     self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1622                     return
1623                 elif n > self._max_yahoo_results:
1624                     self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1625                     n = self._max_yahoo_results
1626                 self._download_n_results(query, n)
1627                 return
1628             except ValueError: # parsing prefix as integer fails
1629                 self._download_n_results(query, 1)
1630                 return
1631
1632     def _download_n_results(self, query, n):
1633         """Downloads a specified number of results for a query"""
1634
1635         video_ids = []
1636         already_seen = set()
1637         pagenum = 1
1638
1639         while True:
1640             self.report_download_page(query, pagenum)
1641             result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum)
1642             request = compat_urllib_request.Request(result_url)
1643             try:
1644                 page = compat_urllib_request.urlopen(request).read()
1645             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1646                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1647                 return
1648
1649             # Extract video identifiers
1650             for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1651                 video_id = mobj.group(1)
1652                 if video_id not in already_seen:
1653                     video_ids.append(video_id)
1654                     already_seen.add(video_id)
1655                     if len(video_ids) == n:
1656                         # Specified n videos reached
1657                         for id in video_ids:
1658                             self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1659                         return
1660
1661             if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1662                 for id in video_ids:
1663                     self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1664                 return
1665
1666             pagenum = pagenum + 1
1667
1668
1669 class YoutubePlaylistIE(InfoExtractor):
1670     """Information Extractor for YouTube playlists."""
1671
1672     _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
1673     _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
1674     _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
1675     _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
1676     IE_NAME = u'youtube:playlist'
1677
1678     def __init__(self, downloader=None):
1679         InfoExtractor.__init__(self, downloader)
1680
1681     def report_download_page(self, playlist_id, pagenum):
1682         """Report attempt to download playlist page with given number."""
1683         self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1684
1685     def _real_extract(self, url):
1686         # Extract playlist id
1687         mobj = re.match(self._VALID_URL, url)
1688         if mobj is None:
1689             self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1690             return
1691
1692         # Single video case
1693         if mobj.group(3) is not None:
1694             self._downloader.download([mobj.group(3)])
1695             return
1696
1697         # Download playlist pages
1698         # prefix is 'p' as default for playlists but there are other types that need extra care
1699         playlist_prefix = mobj.group(1)
1700         if playlist_prefix == 'a':
1701             playlist_access = 'artist'
1702         else:
1703             playlist_prefix = 'p'
1704             playlist_access = 'view_play_list'
1705         playlist_id = mobj.group(2)
1706         video_ids = []
1707         pagenum = 1
1708
1709         while True:
1710             self.report_download_page(playlist_id, pagenum)
1711             url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
1712             request = compat_urllib_request.Request(url)
1713             try:
1714                 page = compat_urllib_request.urlopen(request).read()
1715             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1716                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1717                 return
1718
1719             # Extract video identifiers
1720             ids_in_page = []
1721             for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
1722                 if mobj.group(1) not in ids_in_page:
1723                     ids_in_page.append(mobj.group(1))
1724             video_ids.extend(ids_in_page)
1725
1726             if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1727                 break
1728             pagenum = pagenum + 1
1729
1730         playliststart = self._downloader.params.get('playliststart', 1) - 1
1731         playlistend = self._downloader.params.get('playlistend', -1)
1732         if playlistend == -1:
1733             video_ids = video_ids[playliststart:]
1734         else:
1735             video_ids = video_ids[playliststart:playlistend]
1736
1737         for id in video_ids:
1738             self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1739         return
1740
1741
1742 class YoutubeChannelIE(InfoExtractor):
1743     """Information Extractor for YouTube channels."""
1744
1745     _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
1746     _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1747     _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
1748     IE_NAME = u'youtube:channel'
1749
1750     def report_download_page(self, channel_id, pagenum):
1751         """Report attempt to download channel page with given number."""
1752         self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
1753
1754     def _real_extract(self, url):
1755         # Extract channel id
1756         mobj = re.match(self._VALID_URL, url)
1757         if mobj is None:
1758             self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1759             return
1760
1761         # Download channel pages
1762         channel_id = mobj.group(1)
1763         video_ids = []
1764         pagenum = 1
1765
1766         while True:
1767             self.report_download_page(channel_id, pagenum)
1768             url = self._TEMPLATE_URL % (channel_id, pagenum)
1769             request = compat_urllib_request.Request(url)
1770             try:
1771                 page = compat_urllib_request.urlopen(request).read()
1772             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1773                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1774                 return
1775
1776             # Extract video identifiers
1777             ids_in_page = []
1778             for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
1779                 if mobj.group(1) not in ids_in_page:
1780                     ids_in_page.append(mobj.group(1))
1781             video_ids.extend(ids_in_page)
1782
1783             if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1784                 break
1785             pagenum = pagenum + 1
1786
1787         for id in video_ids:
1788             self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1789         return
1790
1791
1792 class YoutubeUserIE(InfoExtractor):
1793     """Information Extractor for YouTube users."""
1794
1795     _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
1796     _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1797     _GDATA_PAGE_SIZE = 50
1798     _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
1799     _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
1800     IE_NAME = u'youtube:user'
1801
1802     def __init__(self, downloader=None):
1803         InfoExtractor.__init__(self, downloader)
1804
1805     def report_download_page(self, username, start_index):
1806         """Report attempt to download user page."""
1807         self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
1808                 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
1809
1810     def _real_extract(self, url):
1811         # Extract username
1812         mobj = re.match(self._VALID_URL, url)
1813         if mobj is None:
1814             self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1815             return
1816
1817         username = mobj.group(1)
1818
1819         # Download video ids using YouTube Data API. Result size per
1820         # query is limited (currently to 50 videos) so we need to query
1821         # page by page until there are no video ids - it means we got
1822         # all of them.
1823
1824         video_ids = []
1825         pagenum = 0
1826
1827         while True:
1828             start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1829             self.report_download_page(username, start_index)
1830
1831             request = compat_urllib_request.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
1832
1833             try:
1834                 page = compat_urllib_request.urlopen(request).read()
1835             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1836                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1837                 return
1838
1839             # Extract video identifiers
1840             ids_in_page = []
1841
1842             for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1843                 if mobj.group(1) not in ids_in_page:
1844                     ids_in_page.append(mobj.group(1))
1845
1846             video_ids.extend(ids_in_page)
1847
1848             # A little optimization - if current page is not
1849             # "full", ie. does not contain PAGE_SIZE video ids then
1850             # we can assume that this page is the last one - there
1851             # are no more ids on further pages - no need to query
1852             # again.
1853
1854             if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1855                 break
1856
1857             pagenum += 1
1858
1859         all_ids_count = len(video_ids)
1860         playliststart = self._downloader.params.get('playliststart', 1) - 1
1861         playlistend = self._downloader.params.get('playlistend', -1)
1862
1863         if playlistend == -1:
1864             video_ids = video_ids[playliststart:]
1865         else:
1866             video_ids = video_ids[playliststart:playlistend]
1867
1868         self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
1869                 (username, all_ids_count, len(video_ids)))
1870
1871         for video_id in video_ids:
1872             self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
1873
1874
1875 class BlipTVUserIE(InfoExtractor):
1876     """Information Extractor for blip.tv users."""
1877
1878     _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
1879     _PAGE_SIZE = 12
1880     IE_NAME = u'blip.tv:user'
1881
1882     def __init__(self, downloader=None):
1883         InfoExtractor.__init__(self, downloader)
1884
1885     def report_download_page(self, username, pagenum):
1886         """Report attempt to download user page."""
1887         self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
1888                 (self.IE_NAME, username, pagenum))
1889
1890     def _real_extract(self, url):
1891         # Extract username
1892         mobj = re.match(self._VALID_URL, url)
1893         if mobj is None:
1894             self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1895             return
1896
1897         username = mobj.group(1)
1898
1899         page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
1900
1901         request = compat_urllib_request.Request(url)
1902
1903         try:
1904             page = compat_urllib_request.urlopen(request).read().decode('utf-8')
1905             mobj = re.search(r'data-users-id="([^"]+)"', page)
1906             page_base = page_base % mobj.group(1)
1907         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1908             self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1909             return
1910
1911
1912         # Download video ids using BlipTV Ajax calls. Result size per
1913         # query is limited (currently to 12 videos) so we need to query
1914         # page by page until there are no video ids - it means we got
1915         # all of them.
1916
1917         video_ids = []
1918         pagenum = 1
1919
1920         while True:
1921             self.report_download_page(username, pagenum)
1922
1923             request = compat_urllib_request.Request( page_base + "&page=" + str(pagenum) )
1924
1925             try:
1926                 page = compat_urllib_request.urlopen(request).read().decode('utf-8')
1927             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1928                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1929                 return
1930
1931             # Extract video identifiers
1932             ids_in_page = []
1933
1934             for mobj in re.finditer(r'href="/([^"]+)"', page):
1935                 if mobj.group(1) not in ids_in_page:
1936                     ids_in_page.append(unescapeHTML(mobj.group(1)))
1937
1938             video_ids.extend(ids_in_page)
1939
1940             # A little optimization - if current page is not
1941             # "full", ie. does not contain PAGE_SIZE video ids then
1942             # we can assume that this page is the last one - there
1943             # are no more ids on further pages - no need to query
1944             # again.
1945
1946             if len(ids_in_page) < self._PAGE_SIZE:
1947                 break
1948
1949             pagenum += 1
1950
1951         all_ids_count = len(video_ids)
1952         playliststart = self._downloader.params.get('playliststart', 1) - 1
1953         playlistend = self._downloader.params.get('playlistend', -1)
1954
1955         if playlistend == -1:
1956             video_ids = video_ids[playliststart:]
1957         else:
1958             video_ids = video_ids[playliststart:playlistend]
1959
1960         self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
1961                 (self.IE_NAME, username, all_ids_count, len(video_ids)))
1962
1963         for video_id in video_ids:
1964             self._downloader.download([u'http://blip.tv/'+video_id])
1965
1966
1967 class DepositFilesIE(InfoExtractor):
1968     """Information extractor for depositfiles.com"""
1969
1970     _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
1971     IE_NAME = u'DepositFiles'
1972
1973     def __init__(self, downloader=None):
1974         InfoExtractor.__init__(self, downloader)
1975
1976     def report_download_webpage(self, file_id):
1977         """Report webpage download."""
1978         self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
1979
1980     def report_extraction(self, file_id):
1981         """Report information extraction."""
1982         self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
1983
1984     def _real_extract(self, url):
1985         file_id = url.split('/')[-1]
1986         # Rebuild url in english locale
1987         url = 'http://depositfiles.com/en/files/' + file_id
1988
1989         # Retrieve file webpage with 'Free download' button pressed
1990         free_download_indication = { 'gateway_result' : '1' }
1991         request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication))
1992         try:
1993             self.report_download_webpage(file_id)
1994             webpage = compat_urllib_request.urlopen(request).read()
1995         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1996             self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
1997             return
1998
1999         # Search for the real file URL
2000         mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2001         if (mobj is None) or (mobj.group(1) is None):
2002             # Try to figure out reason of the error.
2003             mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2004             if (mobj is not None) and (mobj.group(1) is not None):
2005                 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2006                 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2007             else:
2008                 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2009             return
2010
2011         file_url = mobj.group(1)
2012         file_extension = os.path.splitext(file_url)[1][1:]
2013
2014         # Search for file title
2015         mobj = re.search(r'<b title="(.*?)">', webpage)
2016         if mobj is None:
2017             self._downloader.trouble(u'ERROR: unable to extract title')
2018             return
2019         file_title = mobj.group(1).decode('utf-8')
2020
2021         return [{
2022             'id':       file_id.decode('utf-8'),
2023             'url':      file_url.decode('utf-8'),
2024             'uploader': None,
2025             'upload_date':  None,
2026             'title':    file_title,
2027             'ext':      file_extension.decode('utf-8'),
2028         }]
2029
2030
2031 class FacebookIE(InfoExtractor):
2032     """Information Extractor for Facebook"""
2033
2034     _WORKING = False
2035     _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
2036     _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
2037     _NETRC_MACHINE = 'facebook'
2038     _available_formats = ['video', 'highqual', 'lowqual']
2039     _video_extensions = {
2040         'video': 'mp4',
2041         'highqual': 'mp4',
2042         'lowqual': 'mp4',
2043     }
2044     IE_NAME = u'facebook'
2045
2046     def __init__(self, downloader=None):
2047         InfoExtractor.__init__(self, downloader)
2048
2049     def _reporter(self, message):
2050         """Add header and report message."""
2051         self._downloader.to_screen(u'[facebook] %s' % message)
2052
2053     def report_login(self):
2054         """Report attempt to log in."""
2055         self._reporter(u'Logging in')
2056
2057     def report_video_webpage_download(self, video_id):
2058         """Report attempt to download video webpage."""
2059         self._reporter(u'%s: Downloading video webpage' % video_id)
2060
2061     def report_information_extraction(self, video_id):
2062         """Report attempt to extract video information."""
2063         self._reporter(u'%s: Extracting video information' % video_id)
2064
2065     def _parse_page(self, video_webpage):
2066         """Extract video information from page"""
2067         # General data
2068         data = {'title': r'\("video_title", "(.*?)"\)',
2069             'description': r'<div class="datawrap">(.*?)</div>',
2070             'owner': r'\("video_owner_name", "(.*?)"\)',
2071             'thumbnail':  r'\("thumb_url", "(?P<THUMB>.*?)"\)',
2072             }
2073         video_info = {}
2074         for piece in data.keys():
2075             mobj = re.search(data[piece], video_webpage)
2076             if mobj is not None:
2077                 video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
2078
2079         # Video urls
2080         video_urls = {}
2081         for fmt in self._available_formats:
2082             mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
2083             if mobj is not None:
2084                 # URL is in a Javascript segment inside an escaped Unicode format within
2085                 # the generally utf-8 page
2086                 video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
2087         video_info['video_urls'] = video_urls
2088
2089         return video_info
2090
2091     def _real_initialize(self):
2092         if self._downloader is None:
2093             return
2094
2095         useremail = None
2096         password = None
2097         downloader_params = self._downloader.params
2098
2099         # Attempt to use provided username and password or .netrc data
2100         if downloader_params.get('username', None) is not None:
2101             useremail = downloader_params['username']
2102             password = downloader_params['password']
2103         elif downloader_params.get('usenetrc', False):
2104             try:
2105                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
2106                 if info is not None:
2107                     useremail = info[0]
2108                     password = info[2]
2109                 else:
2110                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
2111             except (IOError, netrc.NetrcParseError) as err:
2112                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
2113                 return
2114
2115         if useremail is None:
2116             return
2117
2118         # Log in
2119         login_form = {
2120             'email': useremail,
2121             'pass': password,
2122             'login': 'Log+In'
2123             }
2124         request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
2125         try:
2126             self.report_login()
2127             login_results = compat_urllib_request.urlopen(request).read()
2128             if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
2129                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
2130                 return
2131         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2132             self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
2133             return
2134
2135     def _real_extract(self, url):
2136         mobj = re.match(self._VALID_URL, url)
2137         if mobj is None:
2138             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2139             return
2140         video_id = mobj.group('ID')
2141
2142         # Get video webpage
2143         self.report_video_webpage_download(video_id)
2144         request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2145         try:
2146             page = compat_urllib_request.urlopen(request)
2147             video_webpage = page.read()
2148         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2149             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2150             return
2151
2152         # Start extracting information
2153         self.report_information_extraction(video_id)
2154
2155         # Extract information
2156         video_info = self._parse_page(video_webpage)
2157
2158         # uploader
2159         if 'owner' not in video_info:
2160             self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2161             return
2162         video_uploader = video_info['owner']
2163
2164         # title
2165         if 'title' not in video_info:
2166             self._downloader.trouble(u'ERROR: unable to extract video title')
2167             return
2168         video_title = video_info['title']
2169         video_title = video_title.decode('utf-8')
2170
2171         # thumbnail image
2172         if 'thumbnail' not in video_info:
2173             self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2174             video_thumbnail = ''
2175         else:
2176             video_thumbnail = video_info['thumbnail']
2177
2178         # upload date
2179         upload_date = None
2180         if 'upload_date' in video_info:
2181             upload_time = video_info['upload_date']
2182             timetuple = email.utils.parsedate_tz(upload_time)
2183             if timetuple is not None:
2184                 try:
2185                     upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2186                 except:
2187                     pass
2188
2189         # description
2190         video_description = video_info.get('description', 'No description available.')
2191
2192         url_map = video_info['video_urls']
2193         if len(url_map.keys()) > 0:
2194             # Decide which formats to download
2195             req_format = self._downloader.params.get('format', None)
2196             format_limit = self._downloader.params.get('format_limit', None)
2197
2198             if format_limit is not None and format_limit in self._available_formats:
2199                 format_list = self._available_formats[self._available_formats.index(format_limit):]
2200             else:
2201                 format_list = self._available_formats
2202             existing_formats = [x for x in format_list if x in url_map]
2203             if len(existing_formats) == 0:
2204                 self._downloader.trouble(u'ERROR: no known formats available for video')
2205                 return
2206             if req_format is None:
2207                 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2208             elif req_format == 'worst':
2209                 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
2210             elif req_format == '-1':
2211                 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2212             else:
2213                 # Specific format
2214                 if req_format not in url_map:
2215                     self._downloader.trouble(u'ERROR: requested format not available')
2216                     return
2217                 video_url_list = [(req_format, url_map[req_format])] # Specific format
2218
2219         results = []
2220         for format_param, video_real_url in video_url_list:
2221             # Extension
2222             video_extension = self._video_extensions.get(format_param, 'mp4')
2223
2224             results.append({
2225                 'id':       video_id.decode('utf-8'),
2226                 'url':      video_real_url.decode('utf-8'),
2227                 'uploader': video_uploader.decode('utf-8'),
2228                 'upload_date':  upload_date,
2229                 'title':    video_title,
2230                 'ext':      video_extension.decode('utf-8'),
2231                 'format':   (format_param is None and u'NA' or format_param.decode('utf-8')),
2232                 'thumbnail':    video_thumbnail.decode('utf-8'),
2233                 'description':  video_description.decode('utf-8'),
2234             })
2235         return results
2236
2237 class BlipTVIE(InfoExtractor):
2238     """Information extractor for blip.tv"""
2239
2240     _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2241     _URL_EXT = r'^.*\.([a-z0-9]+)$'
2242     IE_NAME = u'blip.tv'
2243
2244     def report_extraction(self, file_id):
2245         """Report information extraction."""
2246         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2247
2248     def report_direct_download(self, title):
2249         """Report information extraction."""
2250         self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
2251
2252     def _real_extract(self, url):
2253         mobj = re.match(self._VALID_URL, url)
2254         if mobj is None:
2255             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2256             return
2257
2258         if '?' in url:
2259             cchar = '&'
2260         else:
2261             cchar = '?'
2262         json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2263         request = compat_urllib_request.Request(json_url.encode('utf-8'))
2264         self.report_extraction(mobj.group(1))
2265         info = None
2266         try:
2267             urlh = compat_urllib_request.urlopen(request)
2268             if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
2269                 basename = url.split('/')[-1]
2270                 title,ext = os.path.splitext(basename)
2271                 title = title.decode('UTF-8')
2272                 ext = ext.replace('.', '')
2273                 self.report_direct_download(title)
2274                 info = {
2275                     'id': title,
2276                     'url': url,
2277                     'uploader': None,
2278                     'upload_date': None,
2279                     'title': title,
2280                     'ext': ext,
2281                     'urlhandle': urlh
2282                 }
2283         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2284             self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
2285             return
2286         if info is None: # Regular URL
2287             try:
2288                 json_code = urlh.read()
2289             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2290                 self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
2291                 return
2292
2293             try:
2294                 json_data = json.loads(json_code)
2295                 if 'Post' in json_data:
2296                     data = json_data['Post']
2297                 else:
2298                     data = json_data
2299
2300                 upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2301                 video_url = data['media']['url']
2302                 umobj = re.match(self._URL_EXT, video_url)
2303                 if umobj is None:
2304                     raise ValueError('Can not determine filename extension')
2305                 ext = umobj.group(1)
2306
2307                 info = {
2308                     'id': data['item_id'],
2309                     'url': video_url,
2310                     'uploader': data['display_name'],
2311                     'upload_date': upload_date,
2312                     'title': data['title'],
2313                     'ext': ext,
2314                     'format': data['media']['mimeType'],
2315                     'thumbnail': data['thumbnailUrl'],
2316                     'description': data['description'],
2317                     'player_url': data['embedUrl']
2318                 }
2319             except (ValueError,KeyError) as err:
2320                 self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2321                 return
2322
2323         std_headers['User-Agent'] = 'iTunes/10.6.1'
2324         return [info]
2325
2326
2327 class MyVideoIE(InfoExtractor):
2328     """Information Extractor for myvideo.de."""
2329
2330     _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2331     IE_NAME = u'myvideo'
2332
2333     def __init__(self, downloader=None):
2334         InfoExtractor.__init__(self, downloader)
2335
2336     def report_download_webpage(self, video_id):
2337         """Report webpage download."""
2338         self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2339
2340     def report_extraction(self, video_id):
2341         """Report information extraction."""
2342         self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2343
2344     def _real_extract(self,url):
2345         mobj = re.match(self._VALID_URL, url)
2346         if mobj is None:
2347             self._download.trouble(u'ERROR: invalid URL: %s' % url)
2348             return
2349
2350         video_id = mobj.group(1)
2351
2352         # Get video webpage
2353         request = compat_urllib_request.Request('http://www.myvideo.de/watch/%s' % video_id)
2354         try:
2355             self.report_download_webpage(video_id)
2356             webpage = compat_urllib_request.urlopen(request).read()
2357         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2358             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
2359             return
2360
2361         self.report_extraction(video_id)
2362         mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
2363                  webpage)
2364         if mobj is None:
2365             self._downloader.trouble(u'ERROR: unable to extract media URL')
2366             return
2367         video_url = mobj.group(1) + ('/%s.flv' % video_id)
2368
2369         mobj = re.search('<title>([^<]+)</title>', webpage)
2370         if mobj is None:
2371             self._downloader.trouble(u'ERROR: unable to extract title')
2372             return
2373
2374         video_title = mobj.group(1)
2375
2376         return [{
2377             'id':       video_id,
2378             'url':      video_url,
2379             'uploader': None,
2380             'upload_date':  None,
2381             'title':    video_title,
2382             'ext':      u'flv',
2383         }]
2384
2385 class ComedyCentralIE(InfoExtractor):
2386     """Information extractor for The Daily Show and Colbert Report """
2387
2388     _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
2389     IE_NAME = u'comedycentral'
2390
2391     _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
2392
2393     _video_extensions = {
2394         '3500': 'mp4',
2395         '2200': 'mp4',
2396         '1700': 'mp4',
2397         '1200': 'mp4',
2398         '750': 'mp4',
2399         '400': 'mp4',
2400     }
2401     _video_dimensions = {
2402         '3500': '1280x720',
2403         '2200': '960x540',
2404         '1700': '768x432',
2405         '1200': '640x360',
2406         '750': '512x288',
2407         '400': '384x216',
2408     }
2409
2410     def report_extraction(self, episode_id):
2411         self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
2412
2413     def report_config_download(self, episode_id):
2414         self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
2415
2416     def report_index_download(self, episode_id):
2417         self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
2418
2419     def report_player_url(self, episode_id):
2420         self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
2421
2422
2423     def _print_formats(self, formats):
2424         print('Available formats:')
2425         for x in formats:
2426             print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
2427
2428
2429     def _real_extract(self, url):
2430         mobj = re.match(self._VALID_URL, url)
2431         if mobj is None:
2432             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2433             return
2434
2435         if mobj.group('shortname'):
2436             if mobj.group('shortname') in ('tds', 'thedailyshow'):
2437                 url = u'http://www.thedailyshow.com/full-episodes/'
2438             else:
2439                 url = u'http://www.colbertnation.com/full-episodes/'
2440             mobj = re.match(self._VALID_URL, url)
2441             assert mobj is not None
2442
2443         dlNewest = not mobj.group('episode')
2444         if dlNewest:
2445             epTitle = mobj.group('showname')
2446         else:
2447             epTitle = mobj.group('episode')
2448
2449         req = compat_urllib_request.Request(url)
2450         self.report_extraction(epTitle)
2451         try:
2452             htmlHandle = compat_urllib_request.urlopen(req)
2453             html = htmlHandle.read()
2454         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2455             self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
2456             return
2457         if dlNewest:
2458             url = htmlHandle.geturl()
2459             mobj = re.match(self._VALID_URL, url)
2460             if mobj is None:
2461                 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
2462                 return
2463             if mobj.group('episode') == '':
2464                 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
2465                 return
2466             epTitle = mobj.group('episode')
2467
2468         mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
2469
2470         if len(mMovieParams) == 0:
2471             # The Colbert Report embeds the information in a without
2472             # a URL prefix; so extract the alternate reference
2473             # and then add the URL prefix manually.
2474
2475             altMovieParams = re.findall('data-mgid="([^"]*episode.*?:.*?)"', html)
2476             if len(altMovieParams) == 0:
2477                 self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
2478                 return
2479             else:
2480                 mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
2481
2482         playerUrl_raw = mMovieParams[0][0]
2483         self.report_player_url(epTitle)
2484         try:
2485             urlHandle = compat_urllib_request.urlopen(playerUrl_raw)
2486             playerUrl = urlHandle.geturl()
2487         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2488             self._downloader.trouble(u'ERROR: unable to find out player URL: ' + compat_str(err))
2489             return
2490
2491         uri = mMovieParams[0][1]
2492         indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})
2493         self.report_index_download(epTitle)
2494         try:
2495             indexXml = compat_urllib_request.urlopen(indexUrl).read()
2496         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2497             self._downloader.trouble(u'ERROR: unable to download episode index: ' + compat_str(err))
2498             return
2499
2500         results = []
2501
2502         idoc = xml.etree.ElementTree.fromstring(indexXml)
2503         itemEls = idoc.findall('.//item')
2504         for itemEl in itemEls:
2505             mediaId = itemEl.findall('./guid')[0].text
2506             shortMediaId = mediaId.split(':')[-1]
2507             showId = mediaId.split(':')[-2].replace('.com', '')
2508             officialTitle = itemEl.findall('./title')[0].text
2509             officialDate = itemEl.findall('./pubDate')[0].text
2510
2511             configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
2512                         compat_urllib_parse.urlencode({'uri': mediaId}))
2513             configReq = compat_urllib_request.Request(configUrl)
2514             self.report_config_download(epTitle)
2515             try:
2516                 configXml = compat_urllib_request.urlopen(configReq).read()
2517             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2518                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
2519                 return
2520
2521             cdoc = xml.etree.ElementTree.fromstring(configXml)
2522             turls = []
2523             for rendition in cdoc.findall('.//rendition'):
2524                 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
2525                 turls.append(finfo)
2526
2527             if len(turls) == 0:
2528                 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
2529                 continue
2530
2531             if self._downloader.params.get('listformats', None):
2532                 self._print_formats([i[0] for i in turls])
2533                 return
2534
2535             # For now, just pick the highest bitrate
2536             format,video_url = turls[-1]
2537
2538             # Get the format arg from the arg stream
2539             req_format = self._downloader.params.get('format', None)
2540
2541             # Select format if we can find one
2542             for f,v in turls:
2543                 if f == req_format:
2544                     format, video_url = f, v
2545                     break
2546
2547             # Patch to download from alternative CDN, which does not
2548             # break on current RTMPDump builds
2549             broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
2550             better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
2551
2552             if video_url.startswith(broken_cdn):
2553                 video_url = video_url.replace(broken_cdn, better_cdn)
2554
2555             effTitle = showId + u'-' + epTitle
2556             info = {
2557                 'id': shortMediaId,
2558                 'url': video_url,
2559                 'uploader': showId,
2560                 'upload_date': officialDate,
2561                 'title': effTitle,
2562                 'ext': 'mp4',
2563                 'format': format,
2564                 'thumbnail': None,
2565                 'description': officialTitle,
2566                 'player_url': None #playerUrl
2567             }
2568
2569             results.append(info)
2570
2571         return results
2572
2573
2574 class EscapistIE(InfoExtractor):
2575     """Information extractor for The Escapist """
2576
2577     _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
2578     IE_NAME = u'escapist'
2579
2580     def report_extraction(self, showName):
2581         self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
2582
2583     def report_config_download(self, showName):
2584         self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
2585
2586     def _real_extract(self, url):
2587         mobj = re.match(self._VALID_URL, url)
2588         if mobj is None:
2589             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2590             return
2591         showName = mobj.group('showname')
2592         videoId = mobj.group('episode')
2593
2594         self.report_extraction(showName)
2595         try:
2596             webPage = compat_urllib_request.urlopen(url)
2597             webPageBytes = webPage.read()
2598             m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
2599             webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
2600         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2601             self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err))
2602             return
2603
2604         descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
2605         description = unescapeHTML(descMatch.group(1))
2606         imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
2607         imgUrl = unescapeHTML(imgMatch.group(1))
2608         playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
2609         playerUrl = unescapeHTML(playerUrlMatch.group(1))
2610         configUrlMatch = re.search('config=(.*)$', playerUrl)
2611         configUrl = compat_urllib_parse.unquote(configUrlMatch.group(1))
2612
2613         self.report_config_download(showName)
2614         try:
2615             configJSON = compat_urllib_request.urlopen(configUrl).read()
2616         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2617             self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err))
2618             return
2619
2620         # Technically, it's JavaScript, not JSON
2621         configJSON = configJSON.replace("'", '"')
2622
2623         try:
2624             config = json.loads(configJSON)
2625         except (ValueError,) as err:
2626             self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + compat_str(err))
2627             return
2628
2629         playlist = config['playlist']
2630         videoUrl = playlist[1]['url']
2631
2632         info = {
2633             'id': videoId,
2634             'url': videoUrl,
2635             'uploader': showName,
2636             'upload_date': None,
2637             'title': showName,
2638             'ext': 'flv',
2639             'thumbnail': imgUrl,
2640             'description': description,
2641             'player_url': playerUrl,
2642         }
2643
2644         return [info]
2645
2646
2647 class CollegeHumorIE(InfoExtractor):
2648     """Information extractor for collegehumor.com"""
2649
2650     _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
2651     IE_NAME = u'collegehumor'
2652
2653     def report_webpage(self, video_id):
2654         """Report information extraction."""
2655         self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2656
2657     def report_extraction(self, video_id):
2658         """Report information extraction."""
2659         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2660
2661     def _real_extract(self, url):
2662         mobj = re.match(self._VALID_URL, url)
2663         if mobj is None:
2664             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2665             return
2666         video_id = mobj.group('videoid')
2667
2668         self.report_webpage(video_id)
2669         request = compat_urllib_request.Request(url)
2670         try:
2671             webpage = compat_urllib_request.urlopen(request).read()
2672         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2673             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2674             return
2675
2676         m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
2677         if m is None:
2678             self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
2679             return
2680         internal_video_id = m.group('internalvideoid')
2681
2682         info = {
2683             'id': video_id,
2684             'internal_id': internal_video_id,
2685             'uploader': None,
2686             'upload_date': None,
2687         }
2688
2689         self.report_extraction(video_id)
2690         xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
2691         try:
2692             metaXml = compat_urllib_request.urlopen(xmlUrl).read()
2693         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2694             self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
2695             return
2696
2697         mdoc = xml.etree.ElementTree.fromstring(metaXml)
2698         try:
2699             videoNode = mdoc.findall('./video')[0]
2700             info['description'] = videoNode.findall('./description')[0].text
2701             info['title'] = videoNode.findall('./caption')[0].text
2702             info['url'] = videoNode.findall('./file')[0].text
2703             info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
2704             info['ext'] = info['url'].rpartition('.')[2]
2705         except IndexError:
2706             self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2707             return
2708
2709         return [info]
2710
2711
2712 class XVideosIE(InfoExtractor):
2713     """Information extractor for xvideos.com"""
2714
2715     _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
2716     IE_NAME = u'xvideos'
2717
2718     def report_webpage(self, video_id):
2719         """Report information extraction."""
2720         self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2721
2722     def report_extraction(self, video_id):
2723         """Report information extraction."""
2724         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2725
2726     def _real_extract(self, url):
2727         mobj = re.match(self._VALID_URL, url)
2728         if mobj is None:
2729             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2730             return
2731         video_id = mobj.group(1).decode('utf-8')
2732
2733         self.report_webpage(video_id)
2734
2735         request = compat_urllib_request.Request(r'http://www.xvideos.com/video' + video_id)
2736         try:
2737             webpage = compat_urllib_request.urlopen(request).read()
2738         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2739             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2740             return
2741
2742         self.report_extraction(video_id)
2743
2744
2745         # Extract video URL
2746         mobj = re.search(r'flv_url=(.+?)&', webpage)
2747         if mobj is None:
2748             self._downloader.trouble(u'ERROR: unable to extract video url')
2749             return
2750         video_url = compat_urllib_parse.unquote(mobj.group(1).decode('utf-8'))
2751
2752
2753         # Extract title
2754         mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
2755         if mobj is None:
2756             self._downloader.trouble(u'ERROR: unable to extract video title')
2757             return
2758         video_title = mobj.group(1).decode('utf-8')
2759
2760
2761         # Extract video thumbnail
2762         mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
2763         if mobj is None:
2764             self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
2765             return
2766         video_thumbnail = mobj.group(0).decode('utf-8')
2767
2768         info = {
2769             'id': video_id,
2770             'url': video_url,
2771             'uploader': None,
2772             'upload_date': None,
2773             'title': video_title,
2774             'ext': 'flv',
2775             'thumbnail': video_thumbnail,
2776             'description': None,
2777         }
2778
2779         return [info]
2780
2781
2782 class SoundcloudIE(InfoExtractor):
2783     """Information extractor for soundcloud.com
2784        To access the media, the uid of the song and a stream token
2785        must be extracted from the page source and the script must make
2786        a request to media.soundcloud.com/crossdomain.xml. Then
2787        the media can be grabbed by requesting from an url composed
2788        of the stream token and uid
2789      """
2790
2791     _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
2792     IE_NAME = u'soundcloud'
2793
2794     def __init__(self, downloader=None):
2795         InfoExtractor.__init__(self, downloader)
2796
2797     def report_webpage(self, video_id):
2798         """Report information extraction."""
2799         self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2800
2801     def report_extraction(self, video_id):
2802         """Report information extraction."""
2803         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2804
2805     def _real_extract(self, url):
2806         mobj = re.match(self._VALID_URL, url)
2807         if mobj is None:
2808             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2809             return
2810
2811         # extract uploader (which is in the url)
2812         uploader = mobj.group(1).decode('utf-8')
2813         # extract simple title (uploader + slug of song title)
2814         slug_title =  mobj.group(2).decode('utf-8')
2815         simple_title = uploader + u'-' + slug_title
2816
2817         self.report_webpage('%s/%s' % (uploader, slug_title))
2818
2819         request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
2820         try:
2821             webpage = compat_urllib_request.urlopen(request).read()
2822         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2823             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2824             return
2825
2826         self.report_extraction('%s/%s' % (uploader, slug_title))
2827
2828         # extract uid and stream token that soundcloud hands out for access
2829         mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
2830         if mobj:
2831             video_id = mobj.group(1)
2832             stream_token = mobj.group(2)
2833
2834         # extract unsimplified title
2835         mobj = re.search('"title":"(.*?)",', webpage)
2836         if mobj:
2837             title = mobj.group(1).decode('utf-8')
2838         else:
2839             title = simple_title
2840
2841         # construct media url (with uid/token)
2842         mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
2843         mediaURL = mediaURL % (video_id, stream_token)
2844
2845         # description
2846         description = u'No description available'
2847         mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
2848         if mobj:
2849             description = mobj.group(1)
2850
2851         # upload date
2852         upload_date = None
2853         mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
2854         if mobj:
2855             try:
2856                 upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
2857             except Exception as err:
2858                 self._downloader.to_stderr(compat_str(err))
2859
2860         # for soundcloud, a request to a cross domain is required for cookies
2861         request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
2862
2863         return [{
2864             'id':       video_id.decode('utf-8'),
2865             'url':      mediaURL,
2866             'uploader': uploader.decode('utf-8'),
2867             'upload_date':  upload_date,
2868             'title':    title,
2869             'ext':      u'mp3',
2870             'description': description.decode('utf-8')
2871         }]
2872
2873
2874 class InfoQIE(InfoExtractor):
2875     """Information extractor for infoq.com"""
2876
2877     _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
2878     IE_NAME = u'infoq'
2879
2880     def report_webpage(self, video_id):
2881         """Report information extraction."""
2882         self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2883
2884     def report_extraction(self, video_id):
2885         """Report information extraction."""
2886         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2887
2888     def _real_extract(self, url):
2889         mobj = re.match(self._VALID_URL, url)
2890         if mobj is None:
2891             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2892             return
2893
2894         self.report_webpage(url)
2895
2896         request = compat_urllib_request.Request(url)
2897         try:
2898             webpage = compat_urllib_request.urlopen(request).read()
2899         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2900             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2901             return
2902
2903         self.report_extraction(url)
2904
2905
2906         # Extract video URL
2907         mobj = re.search(r"jsclassref='([^']*)'", webpage)
2908         if mobj is None:
2909             self._downloader.trouble(u'ERROR: unable to extract video url')
2910             return
2911         video_url = 'rtmpe://video.infoq.com/cfx/st/' + compat_urllib_parse.unquote(mobj.group(1).decode('base64'))
2912
2913
2914         # Extract title
2915         mobj = re.search(r'contentTitle = "(.*?)";', webpage)
2916         if mobj is None:
2917             self._downloader.trouble(u'ERROR: unable to extract video title')
2918             return
2919         video_title = mobj.group(1).decode('utf-8')
2920
2921         # Extract description
2922         video_description = u'No description available.'
2923         mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
2924         if mobj is not None:
2925             video_description = mobj.group(1).decode('utf-8')
2926
2927         video_filename = video_url.split('/')[-1]
2928         video_id, extension = video_filename.split('.')
2929
2930         info = {
2931             'id': video_id,
2932             'url': video_url,
2933             'uploader': None,
2934             'upload_date': None,
2935             'title': video_title,
2936             'ext': extension, # Extension is always(?) mp4, but seems to be flv
2937             'thumbnail': None,
2938             'description': video_description,
2939         }
2940
2941         return [info]
2942
2943 class MixcloudIE(InfoExtractor):
2944     """Information extractor for www.mixcloud.com"""
2945     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
2946     IE_NAME = u'mixcloud'
2947
2948     def __init__(self, downloader=None):
2949         InfoExtractor.__init__(self, downloader)
2950
2951     def report_download_json(self, file_id):
2952         """Report JSON download."""
2953         self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
2954
2955     def report_extraction(self, file_id):
2956         """Report information extraction."""
2957         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2958
2959     def get_urls(self, jsonData, fmt, bitrate='best'):
2960         """Get urls from 'audio_formats' section in json"""
2961         file_url = None
2962         try:
2963             bitrate_list = jsonData[fmt]
2964             if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
2965                 bitrate = max(bitrate_list) # select highest
2966
2967             url_list = jsonData[fmt][bitrate]
2968         except TypeError: # we have no bitrate info.
2969             url_list = jsonData[fmt]
2970         return url_list
2971
2972     def check_urls(self, url_list):
2973         """Returns 1st active url from list"""
2974         for url in url_list:
2975             try:
2976                 compat_urllib_request.urlopen(url)
2977                 return url
2978             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2979                 url = None
2980
2981         return None
2982
2983     def _print_formats(self, formats):
2984         print('Available formats:')
2985         for fmt in formats.keys():
2986             for b in formats[fmt]:
2987                 try:
2988                     ext = formats[fmt][b][0]
2989                     print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
2990                 except TypeError: # we have no bitrate info
2991                     ext = formats[fmt][0]
2992                     print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
2993                     break
2994
2995     def _real_extract(self, url):
2996         mobj = re.match(self._VALID_URL, url)
2997         if mobj is None:
2998             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2999             return
3000         # extract uploader & filename from url
3001         uploader = mobj.group(1).decode('utf-8')
3002         file_id = uploader + "-" + mobj.group(2).decode('utf-8')
3003
3004         # construct API request
3005         file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
3006         # retrieve .json file with links to files
3007         request = compat_urllib_request.Request(file_url)
3008         try:
3009             self.report_download_json(file_url)
3010             jsonData = compat_urllib_request.urlopen(request).read()
3011         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3012             self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
3013             return
3014
3015         # parse JSON
3016         json_data = json.loads(jsonData)
3017         player_url = json_data['player_swf_url']
3018         formats = dict(json_data['audio_formats'])
3019
3020         req_format = self._downloader.params.get('format', None)
3021         bitrate = None
3022
3023         if self._downloader.params.get('listformats', None):
3024             self._print_formats(formats)
3025             return
3026
3027         if req_format is None or req_format == 'best':
3028             for format_param in formats.keys():
3029                 url_list = self.get_urls(formats, format_param)
3030                 # check urls
3031                 file_url = self.check_urls(url_list)
3032                 if file_url is not None:
3033                     break # got it!
3034         else:
3035             if req_format not in formats.keys():
3036                 self._downloader.trouble(u'ERROR: format is not available')
3037                 return
3038
3039             url_list = self.get_urls(formats, req_format)
3040             file_url = self.check_urls(url_list)
3041             format_param = req_format
3042
3043         return [{
3044             'id': file_id.decode('utf-8'),
3045             'url': file_url.decode('utf-8'),
3046             'uploader': uploader.decode('utf-8'),
3047             'upload_date': None,
3048             'title': json_data['name'],
3049             'ext': file_url.split('.')[-1].decode('utf-8'),
3050             'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
3051             'thumbnail': json_data['thumbnail_url'],
3052             'description': json_data['description'],
3053             'player_url': player_url.decode('utf-8'),
3054         }]
3055
3056 class StanfordOpenClassroomIE(InfoExtractor):
3057     """Information extractor for Stanford's Open ClassRoom"""
3058
3059     _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
3060     IE_NAME = u'stanfordoc'
3061
3062     def report_download_webpage(self, objid):
3063         """Report information extraction."""
3064         self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
3065
3066     def report_extraction(self, video_id):
3067         """Report information extraction."""
3068         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3069
3070     def _real_extract(self, url):
3071         mobj = re.match(self._VALID_URL, url)
3072         if mobj is None:
3073             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3074             return
3075
3076         if mobj.group('course') and mobj.group('video'): # A specific video
3077             course = mobj.group('course')
3078             video = mobj.group('video')
3079             info = {
3080                 'id': course + '_' + video,
3081                 'uploader': None,
3082                 'upload_date': None,
3083             }
3084
3085             self.report_extraction(info['id'])
3086             baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
3087             xmlUrl = baseUrl + video + '.xml'
3088             try:
3089                 metaXml = compat_urllib_request.urlopen(xmlUrl).read()
3090             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3091                 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
3092                 return
3093             mdoc = xml.etree.ElementTree.fromstring(metaXml)
3094             try:
3095                 info['title'] = mdoc.findall('./title')[0].text
3096                 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
3097             except IndexError:
3098                 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
3099                 return
3100             info['ext'] = info['url'].rpartition('.')[2]
3101             return [info]
3102         elif mobj.group('course'): # A course page
3103             course = mobj.group('course')
3104             info = {
3105                 'id': course,
3106                 'type': 'playlist',
3107                 'uploader': None,
3108                 'upload_date': None,
3109             }
3110
3111             self.report_download_webpage(info['id'])
3112             try:
3113                 coursepage = compat_urllib_request.urlopen(url).read()
3114             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3115                 self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
3116                 return
3117
3118             m = re.search('<h1>([^<]+)</h1>', coursepage)
3119             if m:
3120                 info['title'] = unescapeHTML(m.group(1))
3121             else:
3122                 info['title'] = info['id']
3123
3124             m = re.search('<description>([^<]+)</description>', coursepage)
3125             if m:
3126                 info['description'] = unescapeHTML(m.group(1))
3127
3128             links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
3129             info['list'] = [
3130                 {
3131                     'type': 'reference',
3132                     'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
3133                 }
3134                     for vpage in links]
3135             results = []
3136             for entry in info['list']:
3137                 assert entry['type'] == 'reference'
3138                 results += self.extract(entry['url'])
3139             return results
3140
3141         else: # Root page
3142             info = {
3143                 'id': 'Stanford OpenClassroom',
3144                 'type': 'playlist',
3145                 'uploader': None,
3146                 'upload_date': None,
3147             }
3148
3149             self.report_download_webpage(info['id'])
3150             rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
3151             try:
3152                 rootpage = compat_urllib_request.urlopen(rootURL).read()
3153             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3154                 self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
3155                 return
3156
3157             info['title'] = info['id']
3158
3159             links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
3160             info['list'] = [
3161                 {
3162                     'type': 'reference',
3163                     'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
3164                 }
3165                     for cpage in links]
3166
3167             results = []
3168             for entry in info['list']:
3169                 assert entry['type'] == 'reference'
3170                 results += self.extract(entry['url'])
3171             return results
3172
3173 class MTVIE(InfoExtractor):
3174     """Information extractor for MTV.com"""
3175
3176     _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
3177     IE_NAME = u'mtv'
3178
3179     def report_webpage(self, video_id):
3180         """Report information extraction."""
3181         self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3182
3183     def report_extraction(self, video_id):
3184         """Report information extraction."""
3185         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3186
3187     def _real_extract(self, url):
3188         mobj = re.match(self._VALID_URL, url)
3189         if mobj is None:
3190             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3191             return
3192         if not mobj.group('proto'):
3193             url = 'http://' + url
3194         video_id = mobj.group('videoid')
3195         self.report_webpage(video_id)
3196
3197         request = compat_urllib_request.Request(url)
3198         try:
3199             webpage = compat_urllib_request.urlopen(request).read()
3200         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3201             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
3202             return
3203
3204         mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
3205         if mobj is None:
3206             self._downloader.trouble(u'ERROR: unable to extract song name')
3207             return
3208         song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3209         mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
3210         if mobj is None:
3211             self._downloader.trouble(u'ERROR: unable to extract performer')
3212             return
3213         performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3214         video_title = performer + ' - ' + song_name
3215
3216         mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
3217         if mobj is None:
3218             self._downloader.trouble(u'ERROR: unable to mtvn_uri')
3219             return
3220         mtvn_uri = mobj.group(1)
3221
3222         mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
3223         if mobj is None:
3224             self._downloader.trouble(u'ERROR: unable to extract content id')
3225             return
3226         content_id = mobj.group(1)
3227
3228         videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
3229         self.report_extraction(video_id)
3230         request = compat_urllib_request.Request(videogen_url)
3231         try:
3232             metadataXml = compat_urllib_request.urlopen(request).read()
3233         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3234             self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
3235             return
3236
3237         mdoc = xml.etree.ElementTree.fromstring(metadataXml)
3238         renditions = mdoc.findall('.//rendition')
3239
3240         # For now, always pick the highest quality.
3241         rendition = renditions[-1]
3242
3243         try:
3244             _,_,ext = rendition.attrib['type'].partition('/')
3245             format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
3246             video_url = rendition.find('./src').text
3247         except KeyError:
3248             self._downloader.trouble('Invalid rendition field.')
3249             return
3250
3251         info = {
3252             'id': video_id,
3253             'url': video_url,
3254             'uploader': performer,
3255             'upload_date': None,
3256             'title': video_title,
3257             'ext': ext,
3258             'format': format,
3259         }
3260
3261         return [info]
3262
3263
3264 class YoukuIE(InfoExtractor):
3265
3266     _VALID_URL =  r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
3267     IE_NAME = u'Youku'
3268
3269     def __init__(self, downloader=None):
3270         InfoExtractor.__init__(self, downloader)
3271
3272     def report_download_webpage(self, file_id):
3273         """Report webpage download."""
3274         self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id)
3275
3276     def report_extraction(self, file_id):
3277         """Report information extraction."""
3278         self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id)
3279
3280     def _gen_sid(self):
3281         nowTime = int(time.time() * 1000)
3282         random1 = random.randint(1000,1998)
3283         random2 = random.randint(1000,9999)
3284
3285         return "%d%d%d" %(nowTime,random1,random2)
3286
3287     def _get_file_ID_mix_string(self, seed):
3288         mixed = []
3289         source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
3290         seed = float(seed)
3291         for i in range(len(source)):
3292             seed  =  (seed * 211 + 30031 ) % 65536
3293             index  =  math.floor(seed / 65536 * len(source) )
3294             mixed.append(source[int(index)])
3295             source.remove(source[int(index)])
3296         #return ''.join(mixed)
3297         return mixed
3298
3299     def _get_file_id(self, fileId, seed):
3300         mixed = self._get_file_ID_mix_string(seed)
3301         ids = fileId.split('*')
3302         realId = []
3303         for ch in ids:
3304             if ch:
3305                 realId.append(mixed[int(ch)])
3306         return ''.join(realId)
3307
3308     def _real_extract(self, url):
3309         mobj = re.match(self._VALID_URL, url)
3310         if mobj is None:
3311             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3312             return
3313         video_id = mobj.group('ID')
3314
3315         info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
3316
3317         request = compat_urllib_request.Request(info_url, None, std_headers)
3318         try:
3319             self.report_download_webpage(video_id)
3320             jsondata = compat_urllib_request.urlopen(request).read()
3321         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3322             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
3323             return
3324
3325         self.report_extraction(video_id)
3326         try:
3327             config = json.loads(jsondata)
3328
3329             video_title =  config['data'][0]['title']
3330             seed = config['data'][0]['seed']
3331
3332             format = self._downloader.params.get('format', None)
3333             supported_format = config['data'][0]['streamfileids'].keys()
3334
3335             if format is None or format == 'best':
3336                 if 'hd2' in supported_format:
3337                     format = 'hd2'
3338                 else:
3339                     format = 'flv'
3340                 ext = u'flv'
3341             elif format == 'worst':
3342                 format = 'mp4'
3343                 ext = u'mp4'
3344             else:
3345                 format = 'flv'
3346                 ext = u'flv'
3347
3348
3349             fileid = config['data'][0]['streamfileids'][format]
3350             seg_number = len(config['data'][0]['segs'][format])
3351
3352             keys=[]
3353             for i in xrange(seg_number):
3354                 keys.append(config['data'][0]['segs'][format][i]['k'])
3355
3356             #TODO check error
3357             #youku only could be viewed from mainland china
3358         except:
3359             self._downloader.trouble(u'ERROR: unable to extract info section')
3360             return
3361
3362         files_info=[]
3363         sid = self._gen_sid()
3364         fileid = self._get_file_id(fileid, seed)
3365
3366         #column 8,9 of fileid represent the segment number
3367         #fileid[7:9] should be changed
3368         for index, key in enumerate(keys):
3369
3370             temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
3371             download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
3372
3373             info = {
3374                 'id': '%s_part%02d' % (video_id, index),
3375                 'url': download_url,
3376                 'uploader': None,
3377                 'upload_date': None,
3378                 'title': video_title,
3379                 'ext': ext,
3380             }
3381             files_info.append(info)
3382
3383         return files_info
3384
3385
3386 class XNXXIE(InfoExtractor):
3387     """Information extractor for xnxx.com"""
3388
3389     _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)'
3390     IE_NAME = u'xnxx'
3391     VIDEO_URL_RE = r'flv_url=(.*?)&amp;'
3392     VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
3393     VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'
3394
3395     def report_webpage(self, video_id):
3396         """Report information extraction"""
3397         self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3398
3399     def report_extraction(self, video_id):
3400         """Report information extraction"""
3401         self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3402
3403     def _real_extract(self, url):
3404         mobj = re.match(self._VALID_URL, url)
3405         if mobj is None:
3406             self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3407             return
3408         video_id = mobj.group(1).decode('utf-8')
3409
3410         self.report_webpage(video_id)
3411
3412         # Get webpage content
3413         try:
3414             webpage = compat_urllib_request.urlopen(url).read()
3415         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3416             self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3417             return
3418
3419         result = re.search(self.VIDEO_URL_RE, webpage)
3420         if result is None:
3421             self._downloader.trouble(u'ERROR: unable to extract video url')
3422             return
3423         video_url = compat_urllib_parse.unquote(result.group(1).decode('utf-8'))
3424
3425         result = re.search(self.VIDEO_TITLE_RE, webpage)
3426         if result is None:
3427             self._downloader.trouble(u'ERROR: unable to extract video title')
3428             return
3429         video_title = result.group(1).decode('utf-8')
3430
3431         result = re.search(self.VIDEO_THUMB_RE, webpage)
3432         if result is None:
3433             self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
3434             return
3435         video_thumbnail = result.group(1).decode('utf-8')
3436
3437         return [{
3438             'id': video_id,
3439             'url': video_url,
3440             'uploader': None,
3441             'upload_date': None,
3442             'title': video_title,
3443             'ext': 'flv',
3444             'thumbnail': video_thumbnail,
3445             'description': None,
3446         }]
3447
3448
3449 class GooglePlusIE(InfoExtractor):
3450     """Information extractor for plus.google.com."""
3451
3452     _VALID_URL = r'(?:https://)?plus\.google\.com/(?:\w+/)*?(\d+)/posts/(\w+)'
3453     IE_NAME = u'plus.google'
3454
3455     def __init__(self, downloader=None):
3456         InfoExtractor.__init__(self, downloader)
3457
3458     def report_extract_entry(self, url):
3459         """Report downloading extry"""
3460         self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url.decode('utf-8'))
3461
3462     def report_date(self, upload_date):
3463         """Report downloading extry"""
3464         self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
3465
3466     def report_uploader(self, uploader):
3467         """Report downloading extry"""
3468         self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader.decode('utf-8'))
3469
3470     def report_title(self, video_title):
3471         """Report downloading extry"""
3472         self._downloader.to_screen(u'[plus.google] Title: %s' % video_title.decode('utf-8'))
3473
3474     def report_extract_vid_page(self, video_page):
3475         """Report information extraction."""
3476         self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page.decode('utf-8'))
3477
3478     def _real_extract(self, url):
3479         # Extract id from URL
3480         mobj = re.match(self._VALID_URL, url)
3481         if mobj is None:
3482             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
3483             return
3484
3485         post_url = mobj.group(0)
3486         video_id = mobj.group(2)
3487
3488         video_extension = 'flv'
3489
3490         # Step 1, Retrieve post webpage to extract further information
3491         self.report_extract_entry(post_url)
3492         request = compat_urllib_request.Request(post_url)
3493         try:
3494             webpage = compat_urllib_request.urlopen(request).read()
3495         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3496             self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
3497             return
3498
3499         # Extract update date
3500         upload_date = None
3501         pattern = 'title="Timestamp">(.*?)</a>'
3502         mobj = re.search(pattern, webpage)
3503         if mobj:
3504             upload_date = mobj.group(1)
3505             # Convert timestring to a format suitable for filename
3506             upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
3507             upload_date = upload_date.strftime('%Y%m%d')
3508         self.report_date(upload_date)
3509
3510         # Extract uploader
3511         uploader = None
3512         pattern = r'rel\="author".*?>(.*?)</a>'
3513         mobj = re.search(pattern, webpage)
3514         if mobj:
3515             uploader = mobj.group(1)
3516         self.report_uploader(uploader)
3517
3518         # Extract title
3519         # Get the first line for title
3520         video_title = u'NA'
3521         pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
3522         mobj = re.search(pattern, webpage)
3523         if mobj:
3524             video_title = mobj.group(1)
3525         self.report_title(video_title)
3526
3527         # Step 2, Stimulate clicking the image box to launch video
3528         pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
3529         mobj = re.search(pattern, webpage)
3530         if mobj is None:
3531             self._downloader.trouble(u'ERROR: unable to extract video page URL')
3532
3533         video_page = mobj.group(1)
3534         request = compat_urllib_request.Request(video_page)
3535         try:
3536             webpage = compat_urllib_request.urlopen(request).read()
3537         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3538             self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
3539             return
3540         self.report_extract_vid_page(video_page)
3541
3542
3543         # Extract video links on video page
3544         """Extract video links of all sizes"""
3545         pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
3546         mobj = re.findall(pattern, webpage)
3547         if len(mobj) == 0:
3548             self._downloader.trouble(u'ERROR: unable to extract video links')
3549
3550         # Sort in resolution
3551         links = sorted(mobj)
3552
3553         # Choose the lowest of the sort, i.e. highest resolution
3554         video_url = links[-1]
3555         # Only get the url. The resolution part in the tuple has no use anymore
3556         video_url = video_url[-1]
3557         # Treat escaped \u0026 style hex
3558         video_url = unicode(video_url, "unicode_escape")
3559
3560
3561         return [{
3562             'id':       video_id.decode('utf-8'),
3563             'url':      video_url,
3564             'uploader': uploader.decode('utf-8'),
3565             'upload_date':  upload_date.decode('utf-8'),
3566             'title':    video_title.decode('utf-8'),
3567             'ext':      video_extension.decode('utf-8'),
3568         }]