2 # -*- coding: utf-8 -*-
15 import xml.etree.ElementTree
18 from urlparse import parse_qs
21 import cStringIO as StringIO
28 class InfoExtractor(object):
29 """Information Extractor class.
31 Information extractors are the classes that, given a URL, extract
32 information from the video (or videos) the URL refers to. This
33 information includes the real video URL, the video title and simplified
34 title, author and others. The information is stored in a dictionary
35 which is then passed to the FileDownloader. The FileDownloader
36 processes this information possibly downloading the video to the file
37 system, among other possible outcomes. The dictionaries must include
42 uploader: Nickname of the video uploader.
44 ext: Video filename extension.
46 player_url: SWF Player URL (may be None).
48 The following fields are optional. Their primary purpose is to allow
49 youtube-dl to serve as the backend for a video search function, such
50 as the one in youtube2mp3. They are only used when their respective
51 forced printing functions are called:
53 thumbnail: Full URL to a video thumbnail image.
54 description: One-line video description.
56 Subclasses of this one should re-define the _real_initialize() and
57 _real_extract() methods and define a _VALID_URL regexp.
58 Probably, they should also be added to the list of extractors.
64 def __init__(self, downloader=None):
65 """Constructor. Receives an optional downloader."""
67 self.set_downloader(downloader)
69 def suitable(self, url):
70 """Receives a URL and returns True if suitable for this IE."""
71 return re.match(self._VALID_URL, url) is not None
74 """Initializes an instance (authentication, etc)."""
76 self._real_initialize()
79 def extract(self, url):
80 """Extracts URL information and returns it in list of dicts."""
82 return self._real_extract(url)
84 def set_downloader(self, downloader):
85 """Sets the downloader for this IE."""
86 self._downloader = downloader
88 def _real_initialize(self):
89 """Real initialization process. Redefine in subclasses."""
92 def _real_extract(self, url):
93 """Real extraction process. Redefine in subclasses."""
97 class YoutubeIE(InfoExtractor):
98 """Information extractor for youtube.com."""
102 (?:https?://)? # http(s):// (optional)
103 (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
104 tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains
105 (?:.*?\#/)? # handle anchor (#/) redirect urls
106 (?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs
107 (?: # the various things that can precede the ID:
108 (?:(?:v|embed|e)/) # v/ or embed/ or e/
109 |(?: # or the v= param in all its forms
110 (?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
111 (?:\?|\#!?) # the params delimiter ? or # or #!
112 (?:.+&)? # any other preceding param (like /?s=tuff&v=xxxx)
115 )? # optional -> youtube.com/xxxx is OK
116 )? # all until now is optional -> you can pass the naked ID
117 ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
118 (?(1).+)? # if we found the ID, everything can follow
120 _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
121 _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
122 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
123 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
124 _NETRC_MACHINE = 'youtube'
125 # Listed in order of quality
126 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
127 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
128 _video_extensions = {
134 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
140 _video_dimensions = {
158 def suitable(self, url):
159 """Receives a URL and returns True if suitable for this IE."""
160 return re.match(self._VALID_URL, url, re.VERBOSE) is not None
162 def report_lang(self):
163 """Report attempt to set language."""
164 self._downloader.to_screen(u'[youtube] Setting language')
166 def report_login(self):
167 """Report attempt to log in."""
168 self._downloader.to_screen(u'[youtube] Logging in')
170 def report_age_confirmation(self):
171 """Report attempt to confirm age."""
172 self._downloader.to_screen(u'[youtube] Confirming age')
174 def report_video_webpage_download(self, video_id):
175 """Report attempt to download video webpage."""
176 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
178 def report_video_info_webpage_download(self, video_id):
179 """Report attempt to download video info webpage."""
180 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
182 def report_video_subtitles_download(self, video_id):
183 """Report attempt to download video info webpage."""
184 self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
186 def report_information_extraction(self, video_id):
187 """Report attempt to extract video information."""
188 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
190 def report_unavailable_format(self, video_id, format):
191 """Report extracted video URL."""
192 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
194 def report_rtmp_download(self):
195 """Indicate the download will use the RTMP protocol."""
196 self._downloader.to_screen(u'[youtube] RTMP download detected')
198 def _closed_captions_xml_to_srt(self, xml_string):
200 texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
201 # TODO parse xml instead of regex
202 for n, (start, dur_tag, dur, caption) in enumerate(texts):
203 if not dur: dur = '4'
205 end = start + float(dur)
206 start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
207 end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
208 caption = unescapeHTML(caption)
209 caption = unescapeHTML(caption) # double cycle, intentional
210 srt += str(n+1) + '\n'
211 srt += start + ' --> ' + end + '\n'
212 srt += caption + '\n\n'
215 def _print_formats(self, formats):
216 print('Available formats:')
218 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
220 def _real_initialize(self):
221 if self._downloader is None:
226 downloader_params = self._downloader.params
228 # Attempt to use provided username and password or .netrc data
229 if downloader_params.get('username', None) is not None:
230 username = downloader_params['username']
231 password = downloader_params['password']
232 elif downloader_params.get('usenetrc', False):
234 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
239 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
240 except (IOError, netrc.NetrcParseError), err:
241 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % u(err))
245 request = urllib2.Request(self._LANG_URL)
248 urllib2.urlopen(request).read()
249 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
250 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % u(err))
253 # No authentication to be performed
259 'current_form': 'loginForm',
261 'action_login': 'Log In',
262 'username': username,
263 'password': password,
265 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
268 login_results = urllib2.urlopen(request).read()
269 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
270 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
272 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
273 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % u(err))
279 'action_confirm': 'Confirm',
281 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
283 self.report_age_confirmation()
284 age_results = urllib2.urlopen(request).read()
285 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
286 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % u(err))
289 def _real_extract(self, url):
290 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
291 mobj = re.search(self._NEXT_URL_RE, url)
293 url = 'http://www.youtube.com/' + urllib.unquote(mobj.group(1)).lstrip('/')
295 # Extract video id from URL
296 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
298 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
300 video_id = mobj.group(2)
303 self.report_video_webpage_download(video_id)
304 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
306 video_webpage = urllib2.urlopen(request).read()
307 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
308 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
311 # Attempt to extract SWF player URL
312 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
314 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
319 self.report_video_info_webpage_download(video_id)
320 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
321 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
322 % (video_id, el_type))
323 request = urllib2.Request(video_info_url)
325 video_info_webpage = urllib2.urlopen(request).read()
326 video_info = parse_qs(video_info_webpage)
327 if 'token' in video_info:
329 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
330 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % u(err))
332 if 'token' not in video_info:
333 if 'reason' in video_info:
334 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
336 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
339 # Check for "rental" videos
340 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
341 self._downloader.trouble(u'ERROR: "rental" videos not supported')
344 # Start extracting information
345 self.report_information_extraction(video_id)
348 if 'author' not in video_info:
349 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
351 video_uploader = urllib.unquote_plus(video_info['author'][0])
354 if 'title' not in video_info:
355 self._downloader.trouble(u'ERROR: unable to extract video title')
357 video_title = urllib.unquote_plus(video_info['title'][0])
358 video_title = video_title.decode('utf-8')
361 if 'thumbnail_url' not in video_info:
362 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
364 else: # don't panic if we can't find it
365 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
369 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
371 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
372 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
373 for expression in format_expressions:
375 upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
380 video_description = get_element_by_id("eow-description", video_webpage.decode('utf8'))
381 if video_description: video_description = clean_html(video_description)
382 else: video_description = ''
385 video_subtitles = None
386 if self._downloader.params.get('writesubtitles', False):
388 self.report_video_subtitles_download(video_id)
389 request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
391 srt_list = urllib2.urlopen(request).read()
392 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
393 raise Trouble(u'WARNING: unable to download video subtitles: %s' % u(err))
394 srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
395 srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
396 if not srt_lang_list:
397 raise Trouble(u'WARNING: video has no closed captions')
398 if self._downloader.params.get('subtitleslang', False):
399 srt_lang = self._downloader.params.get('subtitleslang')
400 elif 'en' in srt_lang_list:
403 srt_lang = srt_lang_list.keys()[0]
404 if not srt_lang in srt_lang_list:
405 raise Trouble(u'WARNING: no closed captions found in the specified language')
406 request = urllib2.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
408 srt_xml = urllib2.urlopen(request).read()
409 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
410 raise Trouble(u'WARNING: unable to download video subtitles: %s' % u(err))
412 raise Trouble(u'WARNING: unable to download video subtitles')
413 video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
414 except Trouble as trouble:
415 self._downloader.trouble(trouble[0])
417 if 'length_seconds' not in video_info:
418 self._downloader.trouble(u'WARNING: unable to extract video duration')
421 video_duration = urllib.unquote_plus(video_info['length_seconds'][0])
424 video_token = urllib.unquote_plus(video_info['token'][0])
426 # Decide which formats to download
427 req_format = self._downloader.params.get('format', None)
429 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
430 self.report_rtmp_download()
431 video_url_list = [(None, video_info['conn'][0])]
432 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
433 url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
434 url_data = [parse_qs(uds) for uds in url_data_strs]
435 url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
436 url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
438 format_limit = self._downloader.params.get('format_limit', None)
439 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
440 if format_limit is not None and format_limit in available_formats:
441 format_list = available_formats[available_formats.index(format_limit):]
443 format_list = available_formats
444 existing_formats = [x for x in format_list if x in url_map]
445 if len(existing_formats) == 0:
446 self._downloader.trouble(u'ERROR: no known formats available for video')
448 if self._downloader.params.get('listformats', None):
449 self._print_formats(existing_formats)
451 if req_format is None or req_format == 'best':
452 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
453 elif req_format == 'worst':
454 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
455 elif req_format in ('-1', 'all'):
456 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
458 # Specific formats. We pick the first in a slash-delimeted sequence.
459 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
460 req_formats = req_format.split('/')
461 video_url_list = None
462 for rf in req_formats:
464 video_url_list = [(rf, url_map[rf])]
466 if video_url_list is None:
467 self._downloader.trouble(u'ERROR: requested format not available')
470 self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
474 for format_param, video_real_url in video_url_list:
476 video_extension = self._video_extensions.get(format_param, 'flv')
479 'id': video_id.decode('utf-8'),
480 'url': video_real_url.decode('utf-8'),
481 'uploader': video_uploader.decode('utf-8'),
482 'upload_date': upload_date,
483 'title': video_title,
484 'ext': video_extension.decode('utf-8'),
485 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
486 'thumbnail': video_thumbnail.decode('utf-8'),
487 'description': video_description,
488 'player_url': player_url,
489 'subtitles': video_subtitles,
490 'duration': video_duration
495 class MetacafeIE(InfoExtractor):
496 """Information Extractor for metacafe.com."""
498 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
499 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
500 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
501 IE_NAME = u'metacafe'
503 def __init__(self, downloader=None):
504 InfoExtractor.__init__(self, downloader)
506 def report_disclaimer(self):
507 """Report disclaimer retrieval."""
508 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
510 def report_age_confirmation(self):
511 """Report attempt to confirm age."""
512 self._downloader.to_screen(u'[metacafe] Confirming age')
514 def report_download_webpage(self, video_id):
515 """Report webpage download."""
516 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
518 def report_extraction(self, video_id):
519 """Report information extraction."""
520 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
522 def _real_initialize(self):
523 # Retrieve disclaimer
524 request = urllib2.Request(self._DISCLAIMER)
526 self.report_disclaimer()
527 disclaimer = urllib2.urlopen(request).read()
528 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
529 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % u(err))
535 'submit': "Continue - I'm over 18",
537 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
539 self.report_age_confirmation()
540 disclaimer = urllib2.urlopen(request).read()
541 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
542 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % u(err))
545 def _real_extract(self, url):
546 # Extract id and simplified title from URL
547 mobj = re.match(self._VALID_URL, url)
549 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
552 video_id = mobj.group(1)
554 # Check if video comes from YouTube
555 mobj2 = re.match(r'^yt-(.*)$', video_id)
556 if mobj2 is not None:
557 self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
560 # Retrieve video webpage to extract further information
561 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
563 self.report_download_webpage(video_id)
564 webpage = urllib2.urlopen(request).read()
565 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
566 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % u(err))
569 # Extract URL, uploader and title from webpage
570 self.report_extraction(video_id)
571 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
573 mediaURL = urllib.unquote(mobj.group(1))
574 video_extension = mediaURL[-3:]
576 # Extract gdaKey if available
577 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
581 gdaKey = mobj.group(1)
582 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
584 mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
586 self._downloader.trouble(u'ERROR: unable to extract media URL')
588 vardict = parse_qs(mobj.group(1))
589 if 'mediaData' not in vardict:
590 self._downloader.trouble(u'ERROR: unable to extract media URL')
592 mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
594 self._downloader.trouble(u'ERROR: unable to extract media URL')
596 mediaURL = mobj.group(1).replace('\\/', '/')
597 video_extension = mediaURL[-3:]
598 video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
600 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
602 self._downloader.trouble(u'ERROR: unable to extract title')
604 video_title = mobj.group(1).decode('utf-8')
606 mobj = re.search(r'submitter=(.*?);', webpage)
608 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
610 video_uploader = mobj.group(1)
613 'id': video_id.decode('utf-8'),
614 'url': video_url.decode('utf-8'),
615 'uploader': video_uploader.decode('utf-8'),
616 'upload_date': u'NA',
617 'title': video_title,
618 'ext': video_extension.decode('utf-8'),
624 class DailymotionIE(InfoExtractor):
625 """Information Extractor for Dailymotion"""
627 _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
628 IE_NAME = u'dailymotion'
630 def __init__(self, downloader=None):
631 InfoExtractor.__init__(self, downloader)
633 def report_download_webpage(self, video_id):
634 """Report webpage download."""
635 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
637 def report_extraction(self, video_id):
638 """Report information extraction."""
639 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
641 def _real_extract(self, url):
642 # Extract id and simplified title from URL
643 mobj = re.match(self._VALID_URL, url)
645 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
648 video_id = mobj.group(1).split('_')[0].split('?')[0]
650 video_extension = 'mp4'
652 # Retrieve video webpage to extract further information
653 request = urllib2.Request(url)
654 request.add_header('Cookie', 'family_filter=off')
656 self.report_download_webpage(video_id)
657 webpage = urllib2.urlopen(request).read()
658 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
659 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % u(err))
662 # Extract URL, uploader and title from webpage
663 self.report_extraction(video_id)
664 mobj = re.search(r'\s*var flashvars = (.*)', webpage)
666 self._downloader.trouble(u'ERROR: unable to extract media URL')
668 flashvars = urllib.unquote(mobj.group(1))
670 for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
673 self._downloader.to_screen(u'[dailymotion] Using %s' % key)
676 self._downloader.trouble(u'ERROR: unable to extract video URL')
679 mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
681 self._downloader.trouble(u'ERROR: unable to extract video URL')
684 video_url = urllib.unquote(mobj.group(1)).replace('\\/', '/')
686 # TODO: support choosing qualities
688 mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
690 self._downloader.trouble(u'ERROR: unable to extract title')
692 video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
694 video_uploader = u'NA'
695 mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
697 # lookin for official user
698 mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
699 if mobj_official is None:
700 self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
702 video_uploader = mobj_official.group(1)
704 video_uploader = mobj.group(1)
706 video_upload_date = u'NA'
707 mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
709 video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
712 'id': video_id.decode('utf-8'),
713 'url': video_url.decode('utf-8'),
714 'uploader': video_uploader.decode('utf-8'),
715 'upload_date': video_upload_date,
716 'title': video_title,
717 'ext': video_extension.decode('utf-8'),
723 class GoogleIE(InfoExtractor):
724 """Information extractor for video.google.com."""
726 _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
727 IE_NAME = u'video.google'
729 def __init__(self, downloader=None):
730 InfoExtractor.__init__(self, downloader)
732 def report_download_webpage(self, video_id):
733 """Report webpage download."""
734 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
736 def report_extraction(self, video_id):
737 """Report information extraction."""
738 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
740 def _real_extract(self, url):
741 # Extract id from URL
742 mobj = re.match(self._VALID_URL, url)
744 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
747 video_id = mobj.group(1)
749 video_extension = 'mp4'
751 # Retrieve video webpage to extract further information
752 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
754 self.report_download_webpage(video_id)
755 webpage = urllib2.urlopen(request).read()
756 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
757 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
760 # Extract URL, uploader, and title from webpage
761 self.report_extraction(video_id)
762 mobj = re.search(r"download_url:'([^']+)'", webpage)
764 video_extension = 'flv'
765 mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
767 self._downloader.trouble(u'ERROR: unable to extract media URL')
769 mediaURL = urllib.unquote(mobj.group(1))
770 mediaURL = mediaURL.replace('\\x3d', '\x3d')
771 mediaURL = mediaURL.replace('\\x26', '\x26')
775 mobj = re.search(r'<title>(.*)</title>', webpage)
777 self._downloader.trouble(u'ERROR: unable to extract title')
779 video_title = mobj.group(1).decode('utf-8')
781 # Extract video description
782 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
784 self._downloader.trouble(u'ERROR: unable to extract video description')
786 video_description = mobj.group(1).decode('utf-8')
787 if not video_description:
788 video_description = 'No description available.'
790 # Extract video thumbnail
791 if self._downloader.params.get('forcethumbnail', False):
792 request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
794 webpage = urllib2.urlopen(request).read()
795 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
796 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
798 mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
800 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
802 video_thumbnail = mobj.group(1)
803 else: # we need something to pass to process_info
807 'id': video_id.decode('utf-8'),
808 'url': video_url.decode('utf-8'),
810 'upload_date': u'NA',
811 'title': video_title,
812 'ext': video_extension.decode('utf-8'),
818 class PhotobucketIE(InfoExtractor):
819 """Information extractor for photobucket.com."""
821 _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
822 IE_NAME = u'photobucket'
824 def __init__(self, downloader=None):
825 InfoExtractor.__init__(self, downloader)
827 def report_download_webpage(self, video_id):
828 """Report webpage download."""
829 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
831 def report_extraction(self, video_id):
832 """Report information extraction."""
833 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
835 def _real_extract(self, url):
836 # Extract id from URL
837 mobj = re.match(self._VALID_URL, url)
839 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
842 video_id = mobj.group(1)
844 video_extension = 'flv'
846 # Retrieve video webpage to extract further information
847 request = urllib2.Request(url)
849 self.report_download_webpage(video_id)
850 webpage = urllib2.urlopen(request).read()
851 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
852 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
855 # Extract URL, uploader, and title from webpage
856 self.report_extraction(video_id)
857 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
859 self._downloader.trouble(u'ERROR: unable to extract media URL')
861 mediaURL = urllib.unquote(mobj.group(1))
865 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
867 self._downloader.trouble(u'ERROR: unable to extract title')
869 video_title = mobj.group(1).decode('utf-8')
871 video_uploader = mobj.group(2).decode('utf-8')
874 'id': video_id.decode('utf-8'),
875 'url': video_url.decode('utf-8'),
876 'uploader': video_uploader,
877 'upload_date': u'NA',
878 'title': video_title,
879 'ext': video_extension.decode('utf-8'),
885 class YahooIE(InfoExtractor):
886 """Information extractor for video.yahoo.com."""
888 # _VALID_URL matches all Yahoo! Video URLs
889 # _VPAGE_URL matches only the extractable '/watch/' URLs
890 _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
891 _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
892 IE_NAME = u'video.yahoo'
894 def __init__(self, downloader=None):
895 InfoExtractor.__init__(self, downloader)
897 def report_download_webpage(self, video_id):
898 """Report webpage download."""
899 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
901 def report_extraction(self, video_id):
902 """Report information extraction."""
903 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
905 def _real_extract(self, url, new_video=True):
906 # Extract ID from URL
907 mobj = re.match(self._VALID_URL, url)
909 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
912 video_id = mobj.group(2)
913 video_extension = 'flv'
915 # Rewrite valid but non-extractable URLs as
916 # extractable English language /watch/ URLs
917 if re.match(self._VPAGE_URL, url) is None:
918 request = urllib2.Request(url)
920 webpage = urllib2.urlopen(request).read()
921 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
922 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
925 mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
927 self._downloader.trouble(u'ERROR: Unable to extract id field')
929 yahoo_id = mobj.group(1)
931 mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
933 self._downloader.trouble(u'ERROR: Unable to extract vid field')
935 yahoo_vid = mobj.group(1)
937 url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
938 return self._real_extract(url, new_video=False)
940 # Retrieve video webpage to extract further information
941 request = urllib2.Request(url)
943 self.report_download_webpage(video_id)
944 webpage = urllib2.urlopen(request).read()
945 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
946 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
949 # Extract uploader and title from webpage
950 self.report_extraction(video_id)
951 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
953 self._downloader.trouble(u'ERROR: unable to extract video title')
955 video_title = mobj.group(1).decode('utf-8')
957 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
959 self._downloader.trouble(u'ERROR: unable to extract video uploader')
961 video_uploader = mobj.group(1).decode('utf-8')
963 # Extract video thumbnail
964 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
966 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
968 video_thumbnail = mobj.group(1).decode('utf-8')
970 # Extract video description
971 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
973 self._downloader.trouble(u'ERROR: unable to extract video description')
975 video_description = mobj.group(1).decode('utf-8')
976 if not video_description:
977 video_description = 'No description available.'
979 # Extract video height and width
980 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
982 self._downloader.trouble(u'ERROR: unable to extract video height')
984 yv_video_height = mobj.group(1)
986 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
988 self._downloader.trouble(u'ERROR: unable to extract video width')
990 yv_video_width = mobj.group(1)
992 # Retrieve video playlist to extract media URL
993 # I'm not completely sure what all these options are, but we
994 # seem to need most of them, otherwise the server sends a 401.
995 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
996 yv_bitrate = '700' # according to Wikipedia this is hard-coded
997 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
998 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
999 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1001 self.report_download_webpage(video_id)
1002 webpage = urllib2.urlopen(request).read()
1003 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1004 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
1007 # Extract media URL from playlist XML
1008 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1010 self._downloader.trouble(u'ERROR: Unable to extract media URL')
1012 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1013 video_url = unescapeHTML(video_url)
1016 'id': video_id.decode('utf-8'),
1018 'uploader': video_uploader,
1019 'upload_date': u'NA',
1020 'title': video_title,
1021 'ext': video_extension.decode('utf-8'),
1022 'thumbnail': video_thumbnail.decode('utf-8'),
1023 'description': video_description,
1024 'thumbnail': video_thumbnail,
1029 class VimeoIE(InfoExtractor):
1030 """Information extractor for vimeo.com."""
1032 # _VALID_URL matches Vimeo URLs
1033 _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?:videos?/)?([0-9]+)'
1036 def __init__(self, downloader=None):
1037 InfoExtractor.__init__(self, downloader)
1039 def report_download_webpage(self, video_id):
1040 """Report webpage download."""
1041 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1043 def report_extraction(self, video_id):
1044 """Report information extraction."""
1045 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1047 def _real_extract(self, url, new_video=True):
1048 # Extract ID from URL
1049 mobj = re.match(self._VALID_URL, url)
1051 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1054 video_id = mobj.group(1)
1056 # Retrieve video webpage to extract further information
1057 request = urllib2.Request(url, None, std_headers)
1059 self.report_download_webpage(video_id)
1060 webpage = urllib2.urlopen(request).read()
1061 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1062 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
1065 # Now we begin extracting as much information as we can from what we
1066 # retrieved. First we extract the information common to all extractors,
1067 # and latter we extract those that are Vimeo specific.
1068 self.report_extraction(video_id)
1070 # Extract the config JSON
1071 config = webpage.split(' = {config:')[1].split(',assets:')[0]
1073 config = json.loads(config)
1075 self._downloader.trouble(u'ERROR: unable to extract info section')
1079 video_title = config["video"]["title"]
1082 video_uploader = config["video"]["owner"]["name"]
1084 # Extract video thumbnail
1085 video_thumbnail = config["video"]["thumbnail"]
1087 # Extract video description
1088 video_description = get_element_by_id("description", webpage.decode('utf8'))
1089 if video_description: video_description = clean_html(video_description)
1090 else: video_description = ''
1092 # Extract upload date
1093 video_upload_date = u'NA'
1094 mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
1095 if mobj is not None:
1096 video_upload_date = mobj.group(1)
1098 # Vimeo specific: extract request signature and timestamp
1099 sig = config['request']['signature']
1100 timestamp = config['request']['timestamp']
1102 # Vimeo specific: extract video codec and quality information
1103 # First consider quality, then codecs, then take everything
1104 # TODO bind to format param
1105 codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
1106 files = { 'hd': [], 'sd': [], 'other': []}
1107 for codec_name, codec_extension in codecs:
1108 if codec_name in config["video"]["files"]:
1109 if 'hd' in config["video"]["files"][codec_name]:
1110 files['hd'].append((codec_name, codec_extension, 'hd'))
1111 elif 'sd' in config["video"]["files"][codec_name]:
1112 files['sd'].append((codec_name, codec_extension, 'sd'))
1114 files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0]))
1116 for quality in ('hd', 'sd', 'other'):
1117 if len(files[quality]) > 0:
1118 video_quality = files[quality][0][2]
1119 video_codec = files[quality][0][0]
1120 video_extension = files[quality][0][1]
1121 self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
1124 self._downloader.trouble(u'ERROR: no known codec found')
1127 video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
1128 %(video_id, sig, timestamp, video_quality, video_codec.upper())
1133 'uploader': video_uploader,
1134 'upload_date': video_upload_date,
1135 'title': video_title,
1136 'ext': video_extension,
1137 'thumbnail': video_thumbnail,
1138 'description': video_description,
1143 class ArteTvIE(InfoExtractor):
1144 """arte.tv information extractor."""
1146 _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
1147 _LIVE_URL = r'index-[0-9]+\.html$'
1149 IE_NAME = u'arte.tv'
1151 def __init__(self, downloader=None):
1152 InfoExtractor.__init__(self, downloader)
1154 def report_download_webpage(self, video_id):
1155 """Report webpage download."""
1156 self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
1158 def report_extraction(self, video_id):
1159 """Report information extraction."""
1160 self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
1162 def fetch_webpage(self, url):
1163 self._downloader.increment_downloads()
1164 request = urllib2.Request(url)
1166 self.report_download_webpage(url)
1167 webpage = urllib2.urlopen(request).read()
1168 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1169 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
1171 except ValueError, err:
1172 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1176 def grep_webpage(self, url, regex, regexFlags, matchTuples):
1177 page = self.fetch_webpage(url)
1178 mobj = re.search(regex, page, regexFlags)
1182 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1185 for (i, key, err) in matchTuples:
1186 if mobj.group(i) is None:
1187 self._downloader.trouble(err)
1190 info[key] = mobj.group(i)
1194 def extractLiveStream(self, url):
1195 video_lang = url.split('/')[-4]
1196 info = self.grep_webpage(
1198 r'src="(.*?/videothek_js.*?\.js)',
1201 (1, 'url', u'ERROR: Invalid URL: %s' % url)
1204 http_host = url.split('/')[2]
1205 next_url = 'http://%s%s' % (http_host, urllib.unquote(info.get('url')))
1206 info = self.grep_webpage(
1208 r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
1209 '(http://.*?\.swf).*?' +
1213 (1, 'path', u'ERROR: could not extract video path: %s' % url),
1214 (2, 'player', u'ERROR: could not extract video player: %s' % url),
1215 (3, 'url', u'ERROR: could not extract video url: %s' % url)
1218 video_url = u'%s/%s' % (info.get('url'), info.get('path'))
1220 def extractPlus7Stream(self, url):
1221 video_lang = url.split('/')[-3]
1222 info = self.grep_webpage(
1224 r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
1227 (1, 'url', u'ERROR: Invalid URL: %s' % url)
1230 next_url = urllib.unquote(info.get('url'))
1231 info = self.grep_webpage(
1233 r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
1236 (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
1239 next_url = urllib.unquote(info.get('url'))
1241 info = self.grep_webpage(
1243 r'<video id="(.*?)".*?>.*?' +
1244 '<name>(.*?)</name>.*?' +
1245 '<dateVideo>(.*?)</dateVideo>.*?' +
1246 '<url quality="hd">(.*?)</url>',
1249 (1, 'id', u'ERROR: could not extract video id: %s' % url),
1250 (2, 'title', u'ERROR: could not extract video title: %s' % url),
1251 (3, 'date', u'ERROR: could not extract video date: %s' % url),
1252 (4, 'url', u'ERROR: could not extract video url: %s' % url)
1257 'id': info.get('id'),
1258 'url': urllib.unquote(info.get('url')),
1259 'uploader': u'arte.tv',
1260 'upload_date': info.get('date'),
1261 'title': info.get('title'),
1267 def _real_extract(self, url):
1268 video_id = url.split('/')[-1]
1269 self.report_extraction(video_id)
1271 if re.search(self._LIVE_URL, video_id) is not None:
1272 self.extractLiveStream(url)
1275 info = self.extractPlus7Stream(url)
1280 class GenericIE(InfoExtractor):
1281 """Generic last-resort information extractor."""
1284 IE_NAME = u'generic'
1286 def __init__(self, downloader=None):
1287 InfoExtractor.__init__(self, downloader)
1289 def report_download_webpage(self, video_id):
1290 """Report webpage download."""
1291 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1292 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1294 def report_extraction(self, video_id):
1295 """Report information extraction."""
1296 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1298 def report_following_redirect(self, new_url):
1299 """Report information extraction."""
1300 self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
1302 def _test_redirect(self, url):
1303 """Check if it is a redirect, like url shorteners, in case restart chain."""
1304 class HeadRequest(urllib2.Request):
1305 def get_method(self):
1308 class HEADRedirectHandler(urllib2.HTTPRedirectHandler):
1310 Subclass the HTTPRedirectHandler to make it use our
1311 HeadRequest also on the redirected URL
1313 def redirect_request(self, req, fp, code, msg, headers, newurl):
1314 if code in (301, 302, 303, 307):
1315 newurl = newurl.replace(' ', '%20')
1316 newheaders = dict((k,v) for k,v in req.headers.items()
1317 if k.lower() not in ("content-length", "content-type"))
1318 return HeadRequest(newurl,
1320 origin_req_host=req.get_origin_req_host(),
1323 raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
1325 class HTTPMethodFallback(urllib2.BaseHandler):
1327 Fallback to GET if HEAD is not allowed (405 HTTP error)
1329 def http_error_405(self, req, fp, code, msg, headers):
1333 newheaders = dict((k,v) for k,v in req.headers.items()
1334 if k.lower() not in ("content-length", "content-type"))
1335 return self.parent.open(urllib2.Request(req.get_full_url(),
1337 origin_req_host=req.get_origin_req_host(),
1341 opener = urllib2.OpenerDirector()
1342 for handler in [urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler,
1343 HTTPMethodFallback, HEADRedirectHandler,
1344 urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler]:
1345 opener.add_handler(handler())
1347 response = opener.open(HeadRequest(url))
1348 new_url = response.geturl()
1350 if url == new_url: return False
1352 self.report_following_redirect(new_url)
1353 self._downloader.download([new_url])
1356 def _real_extract(self, url):
1357 if self._test_redirect(url): return
1359 video_id = url.split('/')[-1]
1360 request = urllib2.Request(url)
1362 self.report_download_webpage(video_id)
1363 webpage = urllib2.urlopen(request).read()
1364 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1365 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
1367 except ValueError, err:
1368 # since this is the last-resort InfoExtractor, if
1369 # this error is thrown, it'll be thrown here
1370 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1373 self.report_extraction(video_id)
1374 # Start with something easy: JW Player in SWFObject
1375 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1377 # Broaden the search a little bit
1378 mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1380 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1383 # It's possible that one of the regexes
1384 # matched, but returned an empty group:
1385 if mobj.group(1) is None:
1386 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1389 video_url = urllib.unquote(mobj.group(1))
1390 video_id = os.path.basename(video_url)
1392 # here's a fun little line of code for you:
1393 video_extension = os.path.splitext(video_id)[1][1:]
1394 video_id = os.path.splitext(video_id)[0]
1396 # it's tempting to parse this further, but you would
1397 # have to take into account all the variations like
1398 # Video Title - Site Name
1399 # Site Name | Video Title
1400 # Video Title - Tagline | Site Name
1401 # and so on and so forth; it's just not practical
1402 mobj = re.search(r'<title>(.*)</title>', webpage)
1404 self._downloader.trouble(u'ERROR: unable to extract title')
1406 video_title = mobj.group(1).decode('utf-8')
1408 # video uploader is domain name
1409 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1411 self._downloader.trouble(u'ERROR: unable to extract title')
1413 video_uploader = mobj.group(1).decode('utf-8')
1416 'id': video_id.decode('utf-8'),
1417 'url': video_url.decode('utf-8'),
1418 'uploader': video_uploader,
1419 'upload_date': u'NA',
1420 'title': video_title,
1421 'ext': video_extension.decode('utf-8'),
1427 class YoutubeSearchIE(InfoExtractor):
1428 """Information Extractor for YouTube search queries."""
1429 _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
1430 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1431 _max_youtube_results = 1000
1432 IE_NAME = u'youtube:search'
1434 def __init__(self, downloader=None):
1435 InfoExtractor.__init__(self, downloader)
1437 def report_download_page(self, query, pagenum):
1438 """Report attempt to download search page with given number."""
1439 query = query.decode(preferredencoding())
1440 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1442 def _real_extract(self, query):
1443 mobj = re.match(self._VALID_URL, query)
1445 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1448 prefix, query = query.split(':')
1450 query = query.encode('utf-8')
1452 self._download_n_results(query, 1)
1454 elif prefix == 'all':
1455 self._download_n_results(query, self._max_youtube_results)
1461 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1463 elif n > self._max_youtube_results:
1464 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1465 n = self._max_youtube_results
1466 self._download_n_results(query, n)
1468 except ValueError: # parsing prefix as integer fails
1469 self._download_n_results(query, 1)
1472 def _download_n_results(self, query, n):
1473 """Downloads a specified number of results for a query"""
1479 while (50 * pagenum) < limit:
1480 self.report_download_page(query, pagenum+1)
1481 result_url = self._API_URL % (urllib.quote_plus(query), (50*pagenum)+1)
1482 request = urllib2.Request(result_url)
1484 data = urllib2.urlopen(request).read()
1485 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1486 self._downloader.trouble(u'ERROR: unable to download API page: %s' % u(err))
1488 api_response = json.loads(data)['data']
1490 new_ids = list(video['id'] for video in api_response['items'])
1491 video_ids += new_ids
1493 limit = min(n, api_response['totalItems'])
1496 if len(video_ids) > n:
1497 video_ids = video_ids[:n]
1498 for id in video_ids:
1499 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1503 class GoogleSearchIE(InfoExtractor):
1504 """Information Extractor for Google Video search queries."""
1505 _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
1506 _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1507 _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
1508 _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
1509 _max_google_results = 1000
1510 IE_NAME = u'video.google:search'
1512 def __init__(self, downloader=None):
1513 InfoExtractor.__init__(self, downloader)
1515 def report_download_page(self, query, pagenum):
1516 """Report attempt to download playlist page with given number."""
1517 query = query.decode(preferredencoding())
1518 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1520 def _real_extract(self, query):
1521 mobj = re.match(self._VALID_URL, query)
1523 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1526 prefix, query = query.split(':')
1528 query = query.encode('utf-8')
1530 self._download_n_results(query, 1)
1532 elif prefix == 'all':
1533 self._download_n_results(query, self._max_google_results)
1539 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1541 elif n > self._max_google_results:
1542 self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1543 n = self._max_google_results
1544 self._download_n_results(query, n)
1546 except ValueError: # parsing prefix as integer fails
1547 self._download_n_results(query, 1)
1550 def _download_n_results(self, query, n):
1551 """Downloads a specified number of results for a query"""
1557 self.report_download_page(query, pagenum)
1558 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10)
1559 request = urllib2.Request(result_url)
1561 page = urllib2.urlopen(request).read()
1562 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1563 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
1566 # Extract video identifiers
1567 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1568 video_id = mobj.group(1)
1569 if video_id not in video_ids:
1570 video_ids.append(video_id)
1571 if len(video_ids) == n:
1572 # Specified n videos reached
1573 for id in video_ids:
1574 self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1577 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1578 for id in video_ids:
1579 self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1582 pagenum = pagenum + 1
1585 class YahooSearchIE(InfoExtractor):
1586 """Information Extractor for Yahoo! Video search queries."""
1587 _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
1588 _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1589 _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1590 _MORE_PAGES_INDICATOR = r'\s*Next'
1591 _max_yahoo_results = 1000
1592 IE_NAME = u'video.yahoo:search'
1594 def __init__(self, downloader=None):
1595 InfoExtractor.__init__(self, downloader)
1597 def report_download_page(self, query, pagenum):
1598 """Report attempt to download playlist page with given number."""
1599 query = query.decode(preferredencoding())
1600 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1602 def _real_extract(self, query):
1603 mobj = re.match(self._VALID_URL, query)
1605 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1608 prefix, query = query.split(':')
1610 query = query.encode('utf-8')
1612 self._download_n_results(query, 1)
1614 elif prefix == 'all':
1615 self._download_n_results(query, self._max_yahoo_results)
1621 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1623 elif n > self._max_yahoo_results:
1624 self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1625 n = self._max_yahoo_results
1626 self._download_n_results(query, n)
1628 except ValueError: # parsing prefix as integer fails
1629 self._download_n_results(query, 1)
1632 def _download_n_results(self, query, n):
1633 """Downloads a specified number of results for a query"""
1636 already_seen = set()
1640 self.report_download_page(query, pagenum)
1641 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1642 request = urllib2.Request(result_url)
1644 page = urllib2.urlopen(request).read()
1645 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1646 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
1649 # Extract video identifiers
1650 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1651 video_id = mobj.group(1)
1652 if video_id not in already_seen:
1653 video_ids.append(video_id)
1654 already_seen.add(video_id)
1655 if len(video_ids) == n:
1656 # Specified n videos reached
1657 for id in video_ids:
1658 self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1661 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1662 for id in video_ids:
1663 self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1666 pagenum = pagenum + 1
1669 class YoutubePlaylistIE(InfoExtractor):
1670 """Information Extractor for YouTube playlists."""
1672 _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
1673 _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
1674 _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
1675 _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
1676 IE_NAME = u'youtube:playlist'
1678 def __init__(self, downloader=None):
1679 InfoExtractor.__init__(self, downloader)
1681 def report_download_page(self, playlist_id, pagenum):
1682 """Report attempt to download playlist page with given number."""
1683 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1685 def _real_extract(self, url):
1686 # Extract playlist id
1687 mobj = re.match(self._VALID_URL, url)
1689 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1693 if mobj.group(3) is not None:
1694 self._downloader.download([mobj.group(3)])
1697 # Download playlist pages
1698 # prefix is 'p' as default for playlists but there are other types that need extra care
1699 playlist_prefix = mobj.group(1)
1700 if playlist_prefix == 'a':
1701 playlist_access = 'artist'
1703 playlist_prefix = 'p'
1704 playlist_access = 'view_play_list'
1705 playlist_id = mobj.group(2)
1710 self.report_download_page(playlist_id, pagenum)
1711 url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
1712 request = urllib2.Request(url)
1714 page = urllib2.urlopen(request).read()
1715 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1716 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
1719 # Extract video identifiers
1721 for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
1722 if mobj.group(1) not in ids_in_page:
1723 ids_in_page.append(mobj.group(1))
1724 video_ids.extend(ids_in_page)
1726 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1728 pagenum = pagenum + 1
1730 playliststart = self._downloader.params.get('playliststart', 1) - 1
1731 playlistend = self._downloader.params.get('playlistend', -1)
1732 if playlistend == -1:
1733 video_ids = video_ids[playliststart:]
1735 video_ids = video_ids[playliststart:playlistend]
1737 for id in video_ids:
1738 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1742 class YoutubeChannelIE(InfoExtractor):
1743 """Information Extractor for YouTube channels."""
1745 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
1746 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1747 _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
1748 IE_NAME = u'youtube:channel'
1750 def report_download_page(self, channel_id, pagenum):
1751 """Report attempt to download channel page with given number."""
1752 self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
1754 def _real_extract(self, url):
1755 # Extract channel id
1756 mobj = re.match(self._VALID_URL, url)
1758 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1761 # Download channel pages
1762 channel_id = mobj.group(1)
1767 self.report_download_page(channel_id, pagenum)
1768 url = self._TEMPLATE_URL % (channel_id, pagenum)
1769 request = urllib2.Request(url)
1771 page = urllib2.urlopen(request).read()
1772 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1773 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
1776 # Extract video identifiers
1778 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
1779 if mobj.group(1) not in ids_in_page:
1780 ids_in_page.append(mobj.group(1))
1781 video_ids.extend(ids_in_page)
1783 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1785 pagenum = pagenum + 1
1787 for id in video_ids:
1788 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1792 class YoutubeUserIE(InfoExtractor):
1793 """Information Extractor for YouTube users."""
1795 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
1796 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1797 _GDATA_PAGE_SIZE = 50
1798 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
1799 _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
1800 IE_NAME = u'youtube:user'
1802 def __init__(self, downloader=None):
1803 InfoExtractor.__init__(self, downloader)
1805 def report_download_page(self, username, start_index):
1806 """Report attempt to download user page."""
1807 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
1808 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
1810 def _real_extract(self, url):
1812 mobj = re.match(self._VALID_URL, url)
1814 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1817 username = mobj.group(1)
1819 # Download video ids using YouTube Data API. Result size per
1820 # query is limited (currently to 50 videos) so we need to query
1821 # page by page until there are no video ids - it means we got
1828 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1829 self.report_download_page(username, start_index)
1831 request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
1834 page = urllib2.urlopen(request).read()
1835 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1836 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
1839 # Extract video identifiers
1842 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1843 if mobj.group(1) not in ids_in_page:
1844 ids_in_page.append(mobj.group(1))
1846 video_ids.extend(ids_in_page)
1848 # A little optimization - if current page is not
1849 # "full", ie. does not contain PAGE_SIZE video ids then
1850 # we can assume that this page is the last one - there
1851 # are no more ids on further pages - no need to query
1854 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1859 all_ids_count = len(video_ids)
1860 playliststart = self._downloader.params.get('playliststart', 1) - 1
1861 playlistend = self._downloader.params.get('playlistend', -1)
1863 if playlistend == -1:
1864 video_ids = video_ids[playliststart:]
1866 video_ids = video_ids[playliststart:playlistend]
1868 self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
1869 (username, all_ids_count, len(video_ids)))
1871 for video_id in video_ids:
1872 self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
1875 class BlipTVUserIE(InfoExtractor):
1876 """Information Extractor for blip.tv users."""
1878 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
1880 IE_NAME = u'blip.tv:user'
1882 def __init__(self, downloader=None):
1883 InfoExtractor.__init__(self, downloader)
1885 def report_download_page(self, username, pagenum):
1886 """Report attempt to download user page."""
1887 self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
1888 (self.IE_NAME, username, pagenum))
1890 def _real_extract(self, url):
1892 mobj = re.match(self._VALID_URL, url)
1894 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1897 username = mobj.group(1)
1899 page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
1901 request = urllib2.Request(url)
1904 page = urllib2.urlopen(request).read().decode('utf-8')
1905 mobj = re.search(r'data-users-id="([^"]+)"', page)
1906 page_base = page_base % mobj.group(1)
1907 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1908 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
1912 # Download video ids using BlipTV Ajax calls. Result size per
1913 # query is limited (currently to 12 videos) so we need to query
1914 # page by page until there are no video ids - it means we got
1921 self.report_download_page(username, pagenum)
1923 request = urllib2.Request( page_base + "&page=" + str(pagenum) )
1926 page = urllib2.urlopen(request).read().decode('utf-8')
1927 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1928 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1931 # Extract video identifiers
1934 for mobj in re.finditer(r'href="/([^"]+)"', page):
1935 if mobj.group(1) not in ids_in_page:
1936 ids_in_page.append(unescapeHTML(mobj.group(1)))
1938 video_ids.extend(ids_in_page)
1940 # A little optimization - if current page is not
1941 # "full", ie. does not contain PAGE_SIZE video ids then
1942 # we can assume that this page is the last one - there
1943 # are no more ids on further pages - no need to query
1946 if len(ids_in_page) < self._PAGE_SIZE:
1951 all_ids_count = len(video_ids)
1952 playliststart = self._downloader.params.get('playliststart', 1) - 1
1953 playlistend = self._downloader.params.get('playlistend', -1)
1955 if playlistend == -1:
1956 video_ids = video_ids[playliststart:]
1958 video_ids = video_ids[playliststart:playlistend]
1960 self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
1961 (self.IE_NAME, username, all_ids_count, len(video_ids)))
1963 for video_id in video_ids:
1964 self._downloader.download([u'http://blip.tv/'+video_id])
1967 class DepositFilesIE(InfoExtractor):
1968 """Information extractor for depositfiles.com"""
1970 _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
1971 IE_NAME = u'DepositFiles'
1973 def __init__(self, downloader=None):
1974 InfoExtractor.__init__(self, downloader)
1976 def report_download_webpage(self, file_id):
1977 """Report webpage download."""
1978 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
1980 def report_extraction(self, file_id):
1981 """Report information extraction."""
1982 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
1984 def _real_extract(self, url):
1985 file_id = url.split('/')[-1]
1986 # Rebuild url in english locale
1987 url = 'http://depositfiles.com/en/files/' + file_id
1989 # Retrieve file webpage with 'Free download' button pressed
1990 free_download_indication = { 'gateway_result' : '1' }
1991 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
1993 self.report_download_webpage(file_id)
1994 webpage = urllib2.urlopen(request).read()
1995 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1996 self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % u(err))
1999 # Search for the real file URL
2000 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2001 if (mobj is None) or (mobj.group(1) is None):
2002 # Try to figure out reason of the error.
2003 mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2004 if (mobj is not None) and (mobj.group(1) is not None):
2005 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2006 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2008 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2011 file_url = mobj.group(1)
2012 file_extension = os.path.splitext(file_url)[1][1:]
2014 # Search for file title
2015 mobj = re.search(r'<b title="(.*?)">', webpage)
2017 self._downloader.trouble(u'ERROR: unable to extract title')
2019 file_title = mobj.group(1).decode('utf-8')
2022 'id': file_id.decode('utf-8'),
2023 'url': file_url.decode('utf-8'),
2025 'upload_date': u'NA',
2026 'title': file_title,
2027 'ext': file_extension.decode('utf-8'),
2033 class FacebookIE(InfoExtractor):
2034 """Information Extractor for Facebook"""
2036 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
2037 _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
2038 _NETRC_MACHINE = 'facebook'
2039 _available_formats = ['video', 'highqual', 'lowqual']
2040 _video_extensions = {
2045 IE_NAME = u'facebook'
2047 def __init__(self, downloader=None):
2048 InfoExtractor.__init__(self, downloader)
2050 def _reporter(self, message):
2051 """Add header and report message."""
2052 self._downloader.to_screen(u'[facebook] %s' % message)
2054 def report_login(self):
2055 """Report attempt to log in."""
2056 self._reporter(u'Logging in')
2058 def report_video_webpage_download(self, video_id):
2059 """Report attempt to download video webpage."""
2060 self._reporter(u'%s: Downloading video webpage' % video_id)
2062 def report_information_extraction(self, video_id):
2063 """Report attempt to extract video information."""
2064 self._reporter(u'%s: Extracting video information' % video_id)
2066 def _parse_page(self, video_webpage):
2067 """Extract video information from page"""
2069 data = {'title': r'\("video_title", "(.*?)"\)',
2070 'description': r'<div class="datawrap">(.*?)</div>',
2071 'owner': r'\("video_owner_name", "(.*?)"\)',
2072 'thumbnail': r'\("thumb_url", "(?P<THUMB>.*?)"\)',
2075 for piece in data.keys():
2076 mobj = re.search(data[piece], video_webpage)
2077 if mobj is not None:
2078 video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2082 for fmt in self._available_formats:
2083 mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
2084 if mobj is not None:
2085 # URL is in a Javascript segment inside an escaped Unicode format within
2086 # the generally utf-8 page
2087 video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2088 video_info['video_urls'] = video_urls
2092 def _real_initialize(self):
2093 if self._downloader is None:
2098 downloader_params = self._downloader.params
2100 # Attempt to use provided username and password or .netrc data
2101 if downloader_params.get('username', None) is not None:
2102 useremail = downloader_params['username']
2103 password = downloader_params['password']
2104 elif downloader_params.get('usenetrc', False):
2106 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
2107 if info is not None:
2111 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
2112 except (IOError, netrc.NetrcParseError), err:
2113 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % u(err))
2116 if useremail is None:
2125 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
2128 login_results = urllib2.urlopen(request).read()
2129 if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
2130 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
2132 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2133 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % u(err))
2136 def _real_extract(self, url):
2137 mobj = re.match(self._VALID_URL, url)
2139 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2141 video_id = mobj.group('ID')
2144 self.report_video_webpage_download(video_id)
2145 request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2147 page = urllib2.urlopen(request)
2148 video_webpage = page.read()
2149 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2150 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
2153 # Start extracting information
2154 self.report_information_extraction(video_id)
2156 # Extract information
2157 video_info = self._parse_page(video_webpage)
2160 if 'owner' not in video_info:
2161 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2163 video_uploader = video_info['owner']
2166 if 'title' not in video_info:
2167 self._downloader.trouble(u'ERROR: unable to extract video title')
2169 video_title = video_info['title']
2170 video_title = video_title.decode('utf-8')
2173 if 'thumbnail' not in video_info:
2174 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2175 video_thumbnail = ''
2177 video_thumbnail = video_info['thumbnail']
2181 if 'upload_date' in video_info:
2182 upload_time = video_info['upload_date']
2183 timetuple = email.utils.parsedate_tz(upload_time)
2184 if timetuple is not None:
2186 upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2191 video_description = video_info.get('description', 'No description available.')
2193 url_map = video_info['video_urls']
2194 if len(url_map.keys()) > 0:
2195 # Decide which formats to download
2196 req_format = self._downloader.params.get('format', None)
2197 format_limit = self._downloader.params.get('format_limit', None)
2199 if format_limit is not None and format_limit in self._available_formats:
2200 format_list = self._available_formats[self._available_formats.index(format_limit):]
2202 format_list = self._available_formats
2203 existing_formats = [x for x in format_list if x in url_map]
2204 if len(existing_formats) == 0:
2205 self._downloader.trouble(u'ERROR: no known formats available for video')
2207 if req_format is None:
2208 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2209 elif req_format == 'worst':
2210 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
2211 elif req_format == '-1':
2212 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2215 if req_format not in url_map:
2216 self._downloader.trouble(u'ERROR: requested format not available')
2218 video_url_list = [(req_format, url_map[req_format])] # Specific format
2221 for format_param, video_real_url in video_url_list:
2223 video_extension = self._video_extensions.get(format_param, 'mp4')
2226 'id': video_id.decode('utf-8'),
2227 'url': video_real_url.decode('utf-8'),
2228 'uploader': video_uploader.decode('utf-8'),
2229 'upload_date': upload_date,
2230 'title': video_title,
2231 'ext': video_extension.decode('utf-8'),
2232 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
2233 'thumbnail': video_thumbnail.decode('utf-8'),
2234 'description': video_description.decode('utf-8'),
2239 class BlipTVIE(InfoExtractor):
2240 """Information extractor for blip.tv"""
2242 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2243 _URL_EXT = r'^.*\.([a-z0-9]+)$'
2244 IE_NAME = u'blip.tv'
2246 def report_extraction(self, file_id):
2247 """Report information extraction."""
2248 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2250 def report_direct_download(self, title):
2251 """Report information extraction."""
2252 self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
2254 def _real_extract(self, url):
2255 mobj = re.match(self._VALID_URL, url)
2257 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2264 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2265 request = urllib2.Request(json_url.encode('utf-8'))
2266 self.report_extraction(mobj.group(1))
2269 urlh = urllib2.urlopen(request)
2270 if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
2271 basename = url.split('/')[-1]
2272 title,ext = os.path.splitext(basename)
2273 title = title.decode('UTF-8')
2274 ext = ext.replace('.', '')
2275 self.report_direct_download(title)
2283 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2284 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % u(err))
2286 if info is None: # Regular URL
2288 json_code = urlh.read()
2289 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2290 self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % u(err))
2294 json_data = json.loads(json_code)
2295 if 'Post' in json_data:
2296 data = json_data['Post']
2300 upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2301 video_url = data['media']['url']
2302 umobj = re.match(self._URL_EXT, video_url)
2304 raise ValueError('Can not determine filename extension')
2305 ext = umobj.group(1)
2308 'id': data['item_id'],
2310 'uploader': data['display_name'],
2311 'upload_date': upload_date,
2312 'title': data['title'],
2314 'format': data['media']['mimeType'],
2315 'thumbnail': data['thumbnailUrl'],
2316 'description': data['description'],
2317 'player_url': data['embedUrl']
2319 except (ValueError,KeyError), err:
2320 self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2323 std_headers['User-Agent'] = 'iTunes/10.6.1'
2327 class MyVideoIE(InfoExtractor):
2328 """Information Extractor for myvideo.de."""
2330 _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2331 IE_NAME = u'myvideo'
2333 def __init__(self, downloader=None):
2334 InfoExtractor.__init__(self, downloader)
2336 def report_download_webpage(self, video_id):
2337 """Report webpage download."""
2338 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2340 def report_extraction(self, video_id):
2341 """Report information extraction."""
2342 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2344 def _real_extract(self,url):
2345 mobj = re.match(self._VALID_URL, url)
2347 self._download.trouble(u'ERROR: invalid URL: %s' % url)
2350 video_id = mobj.group(1)
2353 request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
2355 self.report_download_webpage(video_id)
2356 webpage = urllib2.urlopen(request).read()
2357 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2358 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
2361 self.report_extraction(video_id)
2362 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
2365 self._downloader.trouble(u'ERROR: unable to extract media URL')
2367 video_url = mobj.group(1) + ('/%s.flv' % video_id)
2369 mobj = re.search('<title>([^<]+)</title>', webpage)
2371 self._downloader.trouble(u'ERROR: unable to extract title')
2374 video_title = mobj.group(1)
2380 'upload_date': u'NA',
2381 'title': video_title,
2387 class ComedyCentralIE(InfoExtractor):
2388 """Information extractor for The Daily Show and Colbert Report """
2390 _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
2391 IE_NAME = u'comedycentral'
2393 _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
2395 _video_extensions = {
2403 _video_dimensions = {
2412 def report_extraction(self, episode_id):
2413 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
2415 def report_config_download(self, episode_id):
2416 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
2418 def report_index_download(self, episode_id):
2419 self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
2421 def report_player_url(self, episode_id):
2422 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
2425 def _print_formats(self, formats):
2426 print('Available formats:')
2428 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
2431 def _real_extract(self, url):
2432 mobj = re.match(self._VALID_URL, url)
2434 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2437 if mobj.group('shortname'):
2438 if mobj.group('shortname') in ('tds', 'thedailyshow'):
2439 url = u'http://www.thedailyshow.com/full-episodes/'
2441 url = u'http://www.colbertnation.com/full-episodes/'
2442 mobj = re.match(self._VALID_URL, url)
2443 assert mobj is not None
2445 dlNewest = not mobj.group('episode')
2447 epTitle = mobj.group('showname')
2449 epTitle = mobj.group('episode')
2451 req = urllib2.Request(url)
2452 self.report_extraction(epTitle)
2454 htmlHandle = urllib2.urlopen(req)
2455 html = htmlHandle.read()
2456 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2457 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
2460 url = htmlHandle.geturl()
2461 mobj = re.match(self._VALID_URL, url)
2463 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
2465 if mobj.group('episode') == '':
2466 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
2468 epTitle = mobj.group('episode')
2470 mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
2472 if len(mMovieParams) == 0:
2473 # The Colbert Report embeds the information in a without
2474 # a URL prefix; so extract the alternate reference
2475 # and then add the URL prefix manually.
2477 altMovieParams = re.findall('data-mgid="([^"]*episode.*?:.*?)"', html)
2478 if len(altMovieParams) == 0:
2479 self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
2482 mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
2484 playerUrl_raw = mMovieParams[0][0]
2485 self.report_player_url(epTitle)
2487 urlHandle = urllib2.urlopen(playerUrl_raw)
2488 playerUrl = urlHandle.geturl()
2489 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2490 self._downloader.trouble(u'ERROR: unable to find out player URL: ' + u(err))
2493 uri = mMovieParams[0][1]
2494 indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
2495 self.report_index_download(epTitle)
2497 indexXml = urllib2.urlopen(indexUrl).read()
2498 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2499 self._downloader.trouble(u'ERROR: unable to download episode index: ' + u(err))
2504 idoc = xml.etree.ElementTree.fromstring(indexXml)
2505 itemEls = idoc.findall('.//item')
2506 for itemEl in itemEls:
2507 mediaId = itemEl.findall('./guid')[0].text
2508 shortMediaId = mediaId.split(':')[-1]
2509 showId = mediaId.split(':')[-2].replace('.com', '')
2510 officialTitle = itemEl.findall('./title')[0].text
2511 officialDate = itemEl.findall('./pubDate')[0].text
2513 configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
2514 urllib.urlencode({'uri': mediaId}))
2515 configReq = urllib2.Request(configUrl)
2516 self.report_config_download(epTitle)
2518 configXml = urllib2.urlopen(configReq).read()
2519 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2520 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
2523 cdoc = xml.etree.ElementTree.fromstring(configXml)
2525 for rendition in cdoc.findall('.//rendition'):
2526 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
2530 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
2533 if self._downloader.params.get('listformats', None):
2534 self._print_formats([i[0] for i in turls])
2537 # For now, just pick the highest bitrate
2538 format,video_url = turls[-1]
2540 # Get the format arg from the arg stream
2541 req_format = self._downloader.params.get('format', None)
2543 # Select format if we can find one
2546 format, video_url = f, v
2549 # Patch to download from alternative CDN, which does not
2550 # break on current RTMPDump builds
2551 broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
2552 better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
2554 if video_url.startswith(broken_cdn):
2555 video_url = video_url.replace(broken_cdn, better_cdn)
2557 effTitle = showId + u'-' + epTitle
2562 'upload_date': officialDate,
2567 'description': officialTitle,
2568 'player_url': None #playerUrl
2571 results.append(info)
2576 class EscapistIE(InfoExtractor):
2577 """Information extractor for The Escapist """
2579 _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
2580 IE_NAME = u'escapist'
2582 def report_extraction(self, showName):
2583 self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
2585 def report_config_download(self, showName):
2586 self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
2588 def _real_extract(self, url):
2589 mobj = re.match(self._VALID_URL, url)
2591 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2593 showName = mobj.group('showname')
2594 videoId = mobj.group('episode')
2596 self.report_extraction(showName)
2598 webPage = urllib2.urlopen(url)
2599 webPageBytes = webPage.read()
2600 m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
2601 webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
2602 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2603 self._downloader.trouble(u'ERROR: unable to download webpage: ' + u(err))
2606 descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
2607 description = unescapeHTML(descMatch.group(1))
2608 imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
2609 imgUrl = unescapeHTML(imgMatch.group(1))
2610 playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
2611 playerUrl = unescapeHTML(playerUrlMatch.group(1))
2612 configUrlMatch = re.search('config=(.*)$', playerUrl)
2613 configUrl = urllib2.unquote(configUrlMatch.group(1))
2615 self.report_config_download(showName)
2617 configJSON = urllib2.urlopen(configUrl).read()
2618 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2619 self._downloader.trouble(u'ERROR: unable to download configuration: ' + u(err))
2622 # Technically, it's JavaScript, not JSON
2623 configJSON = configJSON.replace("'", '"')
2626 config = json.loads(configJSON)
2627 except (ValueError,), err:
2628 self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + u(err))
2631 playlist = config['playlist']
2632 videoUrl = playlist[1]['url']
2637 'uploader': showName,
2638 'upload_date': None,
2642 'thumbnail': imgUrl,
2643 'description': description,
2644 'player_url': playerUrl,
2650 class CollegeHumorIE(InfoExtractor):
2651 """Information extractor for collegehumor.com"""
2653 _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
2654 IE_NAME = u'collegehumor'
2656 def report_webpage(self, video_id):
2657 """Report information extraction."""
2658 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2660 def report_extraction(self, video_id):
2661 """Report information extraction."""
2662 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2664 def _real_extract(self, url):
2665 mobj = re.match(self._VALID_URL, url)
2667 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2669 video_id = mobj.group('videoid')
2671 self.report_webpage(video_id)
2672 request = urllib2.Request(url)
2674 webpage = urllib2.urlopen(request).read()
2675 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2676 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
2679 m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
2681 self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
2683 internal_video_id = m.group('internalvideoid')
2687 'internal_id': internal_video_id,
2690 self.report_extraction(video_id)
2691 xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
2693 metaXml = urllib2.urlopen(xmlUrl).read()
2694 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2695 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % u(err))
2698 mdoc = xml.etree.ElementTree.fromstring(metaXml)
2700 videoNode = mdoc.findall('./video')[0]
2701 info['description'] = videoNode.findall('./description')[0].text
2702 info['title'] = videoNode.findall('./caption')[0].text
2703 info['url'] = videoNode.findall('./file')[0].text
2704 info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
2705 info['ext'] = info['url'].rpartition('.')[2]
2706 info['format'] = info['ext']
2708 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2714 class XVideosIE(InfoExtractor):
2715 """Information extractor for xvideos.com"""
2717 _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
2718 IE_NAME = u'xvideos'
2720 def report_webpage(self, video_id):
2721 """Report information extraction."""
2722 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2724 def report_extraction(self, video_id):
2725 """Report information extraction."""
2726 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2728 def _real_extract(self, url):
2729 mobj = re.match(self._VALID_URL, url)
2731 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2733 video_id = mobj.group(1).decode('utf-8')
2735 self.report_webpage(video_id)
2737 request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
2739 webpage = urllib2.urlopen(request).read()
2740 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2741 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
2744 self.report_extraction(video_id)
2748 mobj = re.search(r'flv_url=(.+?)&', webpage)
2750 self._downloader.trouble(u'ERROR: unable to extract video url')
2752 video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
2756 mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
2758 self._downloader.trouble(u'ERROR: unable to extract video title')
2760 video_title = mobj.group(1).decode('utf-8')
2763 # Extract video thumbnail
2764 mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
2766 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
2768 video_thumbnail = mobj.group(0).decode('utf-8')
2774 'upload_date': None,
2775 'title': video_title,
2778 'thumbnail': video_thumbnail,
2779 'description': None,
2786 class SoundcloudIE(InfoExtractor):
2787 """Information extractor for soundcloud.com
2788 To access the media, the uid of the song and a stream token
2789 must be extracted from the page source and the script must make
2790 a request to media.soundcloud.com/crossdomain.xml. Then
2791 the media can be grabbed by requesting from an url composed
2792 of the stream token and uid
2795 _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
2796 IE_NAME = u'soundcloud'
2798 def __init__(self, downloader=None):
2799 InfoExtractor.__init__(self, downloader)
2801 def report_webpage(self, video_id):
2802 """Report information extraction."""
2803 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2805 def report_extraction(self, video_id):
2806 """Report information extraction."""
2807 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2809 def _real_extract(self, url):
2810 mobj = re.match(self._VALID_URL, url)
2812 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2815 # extract uploader (which is in the url)
2816 uploader = mobj.group(1).decode('utf-8')
2817 # extract simple title (uploader + slug of song title)
2818 slug_title = mobj.group(2).decode('utf-8')
2819 simple_title = uploader + u'-' + slug_title
2821 self.report_webpage('%s/%s' % (uploader, slug_title))
2823 request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
2825 webpage = urllib2.urlopen(request).read()
2826 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2827 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
2830 self.report_extraction('%s/%s' % (uploader, slug_title))
2832 # extract uid and stream token that soundcloud hands out for access
2833 mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
2835 video_id = mobj.group(1)
2836 stream_token = mobj.group(2)
2838 # extract unsimplified title
2839 mobj = re.search('"title":"(.*?)",', webpage)
2841 title = mobj.group(1).decode('utf-8')
2843 title = simple_title
2845 # construct media url (with uid/token)
2846 mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
2847 mediaURL = mediaURL % (video_id, stream_token)
2850 description = u'No description available'
2851 mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
2853 description = mobj.group(1)
2857 mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
2860 upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
2861 except Exception, e:
2862 self._downloader.to_stderr(u(e))
2864 # for soundcloud, a request to a cross domain is required for cookies
2865 request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
2868 'id': video_id.decode('utf-8'),
2870 'uploader': uploader.decode('utf-8'),
2871 'upload_date': upload_date,
2876 'description': description.decode('utf-8')
2880 class InfoQIE(InfoExtractor):
2881 """Information extractor for infoq.com"""
2883 _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
2886 def report_webpage(self, video_id):
2887 """Report information extraction."""
2888 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2890 def report_extraction(self, video_id):
2891 """Report information extraction."""
2892 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2894 def _real_extract(self, url):
2895 mobj = re.match(self._VALID_URL, url)
2897 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2900 self.report_webpage(url)
2902 request = urllib2.Request(url)
2904 webpage = urllib2.urlopen(request).read()
2905 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2906 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
2909 self.report_extraction(url)
2913 mobj = re.search(r"jsclassref='([^']*)'", webpage)
2915 self._downloader.trouble(u'ERROR: unable to extract video url')
2917 video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
2921 mobj = re.search(r'contentTitle = "(.*?)";', webpage)
2923 self._downloader.trouble(u'ERROR: unable to extract video title')
2925 video_title = mobj.group(1).decode('utf-8')
2927 # Extract description
2928 video_description = u'No description available.'
2929 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
2930 if mobj is not None:
2931 video_description = mobj.group(1).decode('utf-8')
2933 video_filename = video_url.split('/')[-1]
2934 video_id, extension = video_filename.split('.')
2940 'upload_date': None,
2941 'title': video_title,
2943 'format': extension, # Extension is always(?) mp4, but seems to be flv
2945 'description': video_description,
2951 class MixcloudIE(InfoExtractor):
2952 """Information extractor for www.mixcloud.com"""
2953 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
2954 IE_NAME = u'mixcloud'
2956 def __init__(self, downloader=None):
2957 InfoExtractor.__init__(self, downloader)
2959 def report_download_json(self, file_id):
2960 """Report JSON download."""
2961 self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
2963 def report_extraction(self, file_id):
2964 """Report information extraction."""
2965 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2967 def get_urls(self, jsonData, fmt, bitrate='best'):
2968 """Get urls from 'audio_formats' section in json"""
2971 bitrate_list = jsonData[fmt]
2972 if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
2973 bitrate = max(bitrate_list) # select highest
2975 url_list = jsonData[fmt][bitrate]
2976 except TypeError: # we have no bitrate info.
2977 url_list = jsonData[fmt]
2980 def check_urls(self, url_list):
2981 """Returns 1st active url from list"""
2982 for url in url_list:
2984 urllib2.urlopen(url)
2986 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2991 def _print_formats(self, formats):
2992 print('Available formats:')
2993 for fmt in formats.keys():
2994 for b in formats[fmt]:
2996 ext = formats[fmt][b][0]
2997 print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
2998 except TypeError: # we have no bitrate info
2999 ext = formats[fmt][0]
3000 print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
3003 def _real_extract(self, url):
3004 mobj = re.match(self._VALID_URL, url)
3006 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3008 # extract uploader & filename from url
3009 uploader = mobj.group(1).decode('utf-8')
3010 file_id = uploader + "-" + mobj.group(2).decode('utf-8')
3012 # construct API request
3013 file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
3014 # retrieve .json file with links to files
3015 request = urllib2.Request(file_url)
3017 self.report_download_json(file_url)
3018 jsonData = urllib2.urlopen(request).read()
3019 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3020 self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % u(err))
3024 json_data = json.loads(jsonData)
3025 player_url = json_data['player_swf_url']
3026 formats = dict(json_data['audio_formats'])
3028 req_format = self._downloader.params.get('format', None)
3031 if self._downloader.params.get('listformats', None):
3032 self._print_formats(formats)
3035 if req_format is None or req_format == 'best':
3036 for format_param in formats.keys():
3037 url_list = self.get_urls(formats, format_param)
3039 file_url = self.check_urls(url_list)
3040 if file_url is not None:
3043 if req_format not in formats.keys():
3044 self._downloader.trouble(u'ERROR: format is not available')
3047 url_list = self.get_urls(formats, req_format)
3048 file_url = self.check_urls(url_list)
3049 format_param = req_format
3052 'id': file_id.decode('utf-8'),
3053 'url': file_url.decode('utf-8'),
3054 'uploader': uploader.decode('utf-8'),
3055 'upload_date': u'NA',
3056 'title': json_data['name'],
3057 'ext': file_url.split('.')[-1].decode('utf-8'),
3058 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
3059 'thumbnail': json_data['thumbnail_url'],
3060 'description': json_data['description'],
3061 'player_url': player_url.decode('utf-8'),
3064 class StanfordOpenClassroomIE(InfoExtractor):
3065 """Information extractor for Stanford's Open ClassRoom"""
3067 _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
3068 IE_NAME = u'stanfordoc'
3070 def report_download_webpage(self, objid):
3071 """Report information extraction."""
3072 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
3074 def report_extraction(self, video_id):
3075 """Report information extraction."""
3076 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3078 def _real_extract(self, url):
3079 mobj = re.match(self._VALID_URL, url)
3081 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3084 if mobj.group('course') and mobj.group('video'): # A specific video
3085 course = mobj.group('course')
3086 video = mobj.group('video')
3088 'id': course + '_' + video,
3091 self.report_extraction(info['id'])
3092 baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
3093 xmlUrl = baseUrl + video + '.xml'
3095 metaXml = urllib2.urlopen(xmlUrl).read()
3096 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3097 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % u(err))
3099 mdoc = xml.etree.ElementTree.fromstring(metaXml)
3101 info['title'] = mdoc.findall('./title')[0].text
3102 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
3104 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
3106 info['ext'] = info['url'].rpartition('.')[2]
3107 info['format'] = info['ext']
3109 elif mobj.group('course'): # A course page
3110 course = mobj.group('course')
3116 self.report_download_webpage(info['id'])
3118 coursepage = urllib2.urlopen(url).read()
3119 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3120 self._downloader.trouble(u'ERROR: unable to download course info page: ' + u(err))
3123 m = re.search('<h1>([^<]+)</h1>', coursepage)
3125 info['title'] = unescapeHTML(m.group(1))
3127 info['title'] = info['id']
3129 m = re.search('<description>([^<]+)</description>', coursepage)
3131 info['description'] = unescapeHTML(m.group(1))
3133 links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
3136 'type': 'reference',
3137 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
3141 for entry in info['list']:
3142 assert entry['type'] == 'reference'
3143 results += self.extract(entry['url'])
3148 'id': 'Stanford OpenClassroom',
3152 self.report_download_webpage(info['id'])
3153 rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
3155 rootpage = urllib2.urlopen(rootURL).read()
3156 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3157 self._downloader.trouble(u'ERROR: unable to download course info page: ' + u(err))
3160 info['title'] = info['id']
3162 links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
3165 'type': 'reference',
3166 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
3171 for entry in info['list']:
3172 assert entry['type'] == 'reference'
3173 results += self.extract(entry['url'])
3176 class MTVIE(InfoExtractor):
3177 """Information extractor for MTV.com"""
3179 _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
3182 def report_webpage(self, video_id):
3183 """Report information extraction."""
3184 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3186 def report_extraction(self, video_id):
3187 """Report information extraction."""
3188 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3190 def _real_extract(self, url):
3191 mobj = re.match(self._VALID_URL, url)
3193 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3195 if not mobj.group('proto'):
3196 url = 'http://' + url
3197 video_id = mobj.group('videoid')
3198 self.report_webpage(video_id)
3200 request = urllib2.Request(url)
3202 webpage = urllib2.urlopen(request).read()
3203 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3204 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
3207 mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
3209 self._downloader.trouble(u'ERROR: unable to extract song name')
3211 song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3212 mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
3214 self._downloader.trouble(u'ERROR: unable to extract performer')
3216 performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3217 video_title = performer + ' - ' + song_name
3219 mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
3221 self._downloader.trouble(u'ERROR: unable to mtvn_uri')
3223 mtvn_uri = mobj.group(1)
3225 mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
3227 self._downloader.trouble(u'ERROR: unable to extract content id')
3229 content_id = mobj.group(1)
3231 videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
3232 self.report_extraction(video_id)
3233 request = urllib2.Request(videogen_url)
3235 metadataXml = urllib2.urlopen(request).read()
3236 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3237 self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % u(err))
3240 mdoc = xml.etree.ElementTree.fromstring(metadataXml)
3241 renditions = mdoc.findall('.//rendition')
3243 # For now, always pick the highest quality.
3244 rendition = renditions[-1]
3247 _,_,ext = rendition.attrib['type'].partition('/')
3248 format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
3249 video_url = rendition.find('./src').text
3251 self._downloader.trouble('Invalid rendition field.')
3257 'uploader': performer,
3258 'title': video_title,
3266 class YoukuIE(InfoExtractor):
3268 _VALID_URL = r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
3271 def __init__(self, downloader=None):
3272 InfoExtractor.__init__(self, downloader)
3274 def report_download_webpage(self, file_id):
3275 """Report webpage download."""
3276 self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id)
3278 def report_extraction(self, file_id):
3279 """Report information extraction."""
3280 self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id)
3283 nowTime = int(time.time() * 1000)
3284 random1 = random.randint(1000,1998)
3285 random2 = random.randint(1000,9999)
3287 return "%d%d%d" %(nowTime,random1,random2)
3289 def _get_file_ID_mix_string(self, seed):
3291 source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
3293 for i in range(len(source)):
3294 seed = (seed * 211 + 30031 ) % 65536
3295 index = math.floor(seed / 65536 * len(source) )
3296 mixed.append(source[int(index)])
3297 source.remove(source[int(index)])
3298 #return ''.join(mixed)
3301 def _get_file_id(self, fileId, seed):
3302 mixed = self._get_file_ID_mix_string(seed)
3303 ids = fileId.split('*')
3307 realId.append(mixed[int(ch)])
3308 return ''.join(realId)
3310 def _real_extract(self, url):
3311 mobj = re.match(self._VALID_URL, url)
3313 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3315 video_id = mobj.group('ID')
3317 info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
3319 request = urllib2.Request(info_url, None, std_headers)
3321 self.report_download_webpage(video_id)
3322 jsondata = urllib2.urlopen(request).read()
3323 except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
3324 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
3327 self.report_extraction(video_id)
3329 config = json.loads(jsondata)
3331 video_title = config['data'][0]['title']
3332 seed = config['data'][0]['seed']
3334 format = self._downloader.params.get('format', None)
3335 supported_format = config['data'][0]['streamfileids'].keys()
3337 if format is None or format == 'best':
3338 if 'hd2' in supported_format:
3343 elif format == 'worst':
3351 fileid = config['data'][0]['streamfileids'][format]
3352 seg_number = len(config['data'][0]['segs'][format])
3355 for i in xrange(seg_number):
3356 keys.append(config['data'][0]['segs'][format][i]['k'])
3359 #youku only could be viewed from mainland china
3361 self._downloader.trouble(u'ERROR: unable to extract info section')
3365 sid = self._gen_sid()
3366 fileid = self._get_file_id(fileid, seed)
3368 #column 8,9 of fileid represent the segment number
3369 #fileid[7:9] should be changed
3370 for index, key in enumerate(keys):
3372 temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
3373 download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
3376 'id': '%s_part%02d' % (video_id, index),
3377 'url': download_url,
3379 'title': video_title,
3383 files_info.append(info)
3388 class XNXXIE(InfoExtractor):
3389 """Information extractor for xnxx.com"""
3391 _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)'
3393 VIDEO_URL_RE = r'flv_url=(.*?)&'
3394 VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
3395 VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&'
3397 def report_webpage(self, video_id):
3398 """Report information extraction"""
3399 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3401 def report_extraction(self, video_id):
3402 """Report information extraction"""
3403 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3405 def _real_extract(self, url):
3406 mobj = re.match(self._VALID_URL, url)
3408 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3410 video_id = mobj.group(1).decode('utf-8')
3412 self.report_webpage(video_id)
3414 # Get webpage content
3416 webpage = urllib2.urlopen(url).read()
3417 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3418 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3421 result = re.search(self.VIDEO_URL_RE, webpage)
3423 self._downloader.trouble(u'ERROR: unable to extract video url')
3425 video_url = urllib.unquote(result.group(1).decode('utf-8'))
3427 result = re.search(self.VIDEO_TITLE_RE, webpage)
3429 self._downloader.trouble(u'ERROR: unable to extract video title')
3431 video_title = result.group(1).decode('utf-8')
3433 result = re.search(self.VIDEO_THUMB_RE, webpage)
3435 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
3437 video_thumbnail = result.group(1).decode('utf-8')
3439 info = {'id': video_id,
3442 'upload_date': None,
3443 'title': video_title,
3446 'thumbnail': video_thumbnail,
3447 'description': None,
3453 class GooglePlusIE(InfoExtractor):
3454 """Information extractor for plus.google.com."""
3456 _VALID_URL = r'(?:https://)?plus\.google\.com/(?:\w+/)*?(\d+)/posts/(\w+)'
3457 IE_NAME = u'plus.google'
3459 def __init__(self, downloader=None):
3460 InfoExtractor.__init__(self, downloader)
3462 def report_extract_entry(self, url):
3463 """Report downloading extry"""
3464 self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url.decode('utf-8'))
3466 def report_date(self, upload_date):
3467 """Report downloading extry"""
3468 self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
3470 def report_uploader(self, uploader):
3471 """Report downloading extry"""
3472 self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader.decode('utf-8'))
3474 def report_title(self, video_title):
3475 """Report downloading extry"""
3476 self._downloader.to_screen(u'[plus.google] Title: %s' % video_title.decode('utf-8'))
3478 def report_extract_vid_page(self, video_page):
3479 """Report information extraction."""
3480 self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page.decode('utf-8'))
3482 def _real_extract(self, url):
3483 # Extract id from URL
3484 mobj = re.match(self._VALID_URL, url)
3486 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
3489 post_url = mobj.group(0)
3490 video_id = mobj.group(2)
3492 video_extension = 'flv'
3494 # Step 1, Retrieve post webpage to extract further information
3495 self.report_extract_entry(post_url)
3496 request = urllib2.Request(post_url)
3498 webpage = urllib2.urlopen(request).read()
3499 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3500 self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % u(err))
3503 # Extract update date
3505 pattern = 'title="Timestamp">(.*?)</a>'
3506 mobj = re.search(pattern, webpage)
3508 upload_date = mobj.group(1)
3509 # Convert timestring to a format suitable for filename
3510 upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
3511 upload_date = upload_date.strftime('%Y%m%d')
3512 self.report_date(upload_date)
3516 pattern = r'rel\="author".*?>(.*?)</a>'
3517 mobj = re.search(pattern, webpage)
3519 uploader = mobj.group(1)
3520 self.report_uploader(uploader)
3523 # Get the first line for title
3525 pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
3526 mobj = re.search(pattern, webpage)
3528 video_title = mobj.group(1)
3529 self.report_title(video_title)
3531 # Step 2, Stimulate clicking the image box to launch video
3532 pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
3533 mobj = re.search(pattern, webpage)
3535 self._downloader.trouble(u'ERROR: unable to extract video page URL')
3537 video_page = mobj.group(1)
3538 request = urllib2.Request(video_page)
3540 webpage = urllib2.urlopen(request).read()
3541 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3542 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
3544 self.report_extract_vid_page(video_page)
3547 # Extract video links on video page
3548 """Extract video links of all sizes"""
3549 pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
3550 mobj = re.findall(pattern, webpage)
3552 self._downloader.trouble(u'ERROR: unable to extract video links')
3554 # Sort in resolution
3555 links = sorted(mobj)
3557 # Choose the lowest of the sort, i.e. highest resolution
3558 video_url = links[-1]
3559 # Only get the url. The resolution part in the tuple has no use anymore
3560 video_url = video_url[-1]
3561 # Treat escaped \u0026 style hex
3562 video_url = unicode(video_url, "unicode_escape")
3566 'id': video_id.decode('utf-8'),
3568 'uploader': uploader.decode('utf-8'),
3569 'upload_date': upload_date.decode('utf-8'),
3570 'title': video_title.decode('utf-8'),
3571 'ext': video_extension.decode('utf-8'),