X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=b4c86cfa311b8391c40bf6e5f0f179724fae7057;hb=46b9d8295d43bb4a5954da95293ef2490eb28b5e;hp=72ad25ad316967a3e85e554fd650e6a76cd3a092;hpb=991ba7fae38498b1066c7f6821d1d8681ef87f1b;p=youtube-dl diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 72ad25ad3..b4c86cfa3 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import base64 import datetime +import itertools import netrc import os import re @@ -14,10 +15,7 @@ import email.utils import xml.etree.ElementTree import random import math -import urllib -import urllib2 -import httplib -from urlparse import parse_qs, urlparse +import operator from .utils import * @@ -39,17 +37,18 @@ class InfoExtractor(object): url: Final video URL. title: Video title, unescaped. ext: Video filename extension. - uploader: Full name of the video uploader. - upload_date: Video upload date (YYYYMMDD). The following fields are optional: format: The video format, defaults to ext (used for --get-format) thumbnail: Full URL to a video thumbnail image. description: One-line video description. + uploader: Full name of the video uploader. + upload_date: Video upload date (YYYYMMDD). uploader_id: Nickname or id of the video uploader. + location: Physical location of the video. player_url: SWF Player URL (used for rtmpdump). - subtitles: The .srt file contents. + subtitles: The subtitle file contents. urlhandle: [internal] The urlHandle to be used to download the file, like returned by urllib.request.urlopen @@ -75,13 +74,15 @@ class InfoExtractor(object): self._ready = False self.set_downloader(downloader) - def suitable(self, url): + @classmethod + def suitable(cls, url): """Receives a URL and returns True if suitable for this IE.""" - return re.match(self._VALID_URL, url) is not None + return re.match(cls._VALID_URL, url) is not None - def working(self): + @classmethod + def working(cls): """Getter method for _WORKING.""" - return self._WORKING + return cls._WORKING def initialize(self): """Initializes an instance (authentication, etc).""" @@ -110,19 +111,30 @@ class InfoExtractor(object): def IE_NAME(self): return type(self).__name__[:-2] - def _download_webpage(self, url_or_request, video_id, note=None, errnote=None): + def _request_webpage(self, url_or_request, video_id, note=None, errnote=None): + """ Returns the response handle """ if note is None: note = u'Downloading video webpage' self._downloader.to_screen(u'[%s] %s: %s' % (self.IE_NAME, video_id, note)) try: - urlh = compat_urllib_request.urlopen(url_or_request) - webpage_bytes = urlh.read() - return webpage_bytes.decode('utf-8', 'replace') + return compat_urllib_request.urlopen(url_or_request) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: if errnote is None: errnote = u'Unable to download webpage' raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2]) + def _download_webpage(self, url_or_request, video_id, note=None, errnote=None): + """ Returns the data of the page as a string """ + urlh = self._request_webpage(url_or_request, video_id, note, errnote) + content_type = urlh.headers.get('Content-Type', '') + m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type) + if m: + encoding = m.group(1) + else: + encoding = 'utf-8' + webpage_bytes = urlh.read() + return webpage_bytes.decode(encoding, 'replace') + class YoutubeIE(InfoExtractor): """Information extractor for youtube.com.""" @@ -133,7 +145,6 @@ class YoutubeIE(InfoExtractor): (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/| tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains (?:.*?\#/)? # handle anchor (#/) redirect urls - (?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs (?: # the various things that can precede the ID: (?:(?:v|embed|e)/) # v/ or embed/ or e/ |(?: # or the v= param in all its forms @@ -148,7 +159,7 @@ class YoutubeIE(InfoExtractor): (?(1).+)? # if we found the ID, everything can follow $""" _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' - _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en' + _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' _NEXT_URL_RE = r'[\?&]next_url=([^&]+)' _NETRC_MACHINE = 'youtube' @@ -185,9 +196,11 @@ class YoutubeIE(InfoExtractor): } IE_NAME = u'youtube' - def suitable(self, url): + @classmethod + def suitable(cls, url): """Receives a URL and returns True if suitable for this IE.""" - return re.match(self._VALID_URL, url, re.VERBOSE) is not None + if YoutubePlaylistIE.suitable(url): return False + return re.match(cls._VALID_URL, url, re.VERBOSE) is not None def report_lang(self): """Report attempt to set language.""" @@ -211,7 +224,16 @@ class YoutubeIE(InfoExtractor): def report_video_subtitles_download(self, video_id): """Report attempt to download video info webpage.""" - self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id) + self._downloader.to_screen(u'[youtube] %s: Checking available subtitles' % video_id) + + def report_video_subtitles_request(self, video_id, sub_lang, format): + """Report attempt to download video info webpage.""" + self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles for %s.%s' % (video_id, sub_lang, format)) + + def report_video_subtitles_available(self, video_id, sub_lang_list): + """Report available subtitles.""" + sub_lang = ",".join(list(sub_lang_list.keys())) + self._downloader.to_screen(u'[youtube] %s: Available subtitles for video: %s' % (video_id, sub_lang)) def report_information_extraction(self, video_id): """Report attempt to extract video information.""" @@ -225,50 +247,63 @@ class YoutubeIE(InfoExtractor): """Indicate the download will use the RTMP protocol.""" self._downloader.to_screen(u'[youtube] RTMP download detected') - def _closed_captions_xml_to_srt(self, xml_string): - srt = '' - texts = re.findall(r'([^<]+)', xml_string, re.MULTILINE) - # TODO parse xml instead of regex - for n, (start, dur_tag, dur, caption) in enumerate(texts): - if not dur: dur = '4' - start = float(start) - end = start + float(dur) - start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000) - end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000) - caption = unescapeHTML(caption) - caption = unescapeHTML(caption) # double cycle, intentional - srt += str(n+1) + '\n' - srt += start + ' --> ' + end + '\n' - srt += caption + '\n\n' - return srt - - def _extract_subtitles(self, video_id): + def _get_available_subtitles(self, video_id): self.report_video_subtitles_download(video_id) request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) try: - srt_list = compat_urllib_request.urlopen(request).read().decode('utf-8') + sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None) - srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list) - srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list) - if not srt_lang_list: - return (u'WARNING: video has no closed captions', None) - if self._downloader.params.get('subtitleslang', False): - srt_lang = self._downloader.params.get('subtitleslang') - elif 'en' in srt_lang_list: - srt_lang = 'en' - else: - srt_lang = list(srt_lang_list.keys())[0] - if not srt_lang in srt_lang_list: - return (u'WARNING: no closed captions found in the specified language', None) - request = compat_urllib_request.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id)) + sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list) + sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list) + if not sub_lang_list: + return (u'WARNING: video doesn\'t have subtitles', None) + return sub_lang_list + + def _list_available_subtitles(self, video_id): + sub_lang_list = self._get_available_subtitles(video_id) + self.report_video_subtitles_available(video_id, sub_lang_list) + + def _request_subtitle(self, sub_lang, sub_name, video_id, format): + self.report_video_subtitles_request(video_id, sub_lang, format) + params = compat_urllib_parse.urlencode({ + 'lang': sub_lang, + 'name': sub_name, + 'v': video_id, + 'fmt': format, + }) + url = 'http://www.youtube.com/api/timedtext?' + params try: - srt_xml = compat_urllib_request.urlopen(request).read().decode('utf-8') + sub = compat_urllib_request.urlopen(url).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None) - if not srt_xml: - return (u'WARNING: unable to download video subtitles', None) - return (None, self._closed_captions_xml_to_srt(srt_xml)) + if not sub: + return (u'WARNING: Did not fetch video subtitles', None) + return (None, sub_lang, sub) + + def _extract_subtitle(self, video_id): + sub_lang_list = self._get_available_subtitles(video_id) + sub_format = self._downloader.params.get('subtitlesformat') + if self._downloader.params.get('subtitleslang', False): + sub_lang = self._downloader.params.get('subtitleslang') + elif 'en' in sub_lang_list: + sub_lang = 'en' + else: + sub_lang = list(sub_lang_list.keys())[0] + if not sub_lang in sub_lang_list: + return (u'WARNING: no closed captions found in the specified language "%s"' % sub_lang, None) + + subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) + return [subtitle] + + def _extract_all_subtitles(self, video_id): + sub_lang_list = self._get_available_subtitles(video_id) + sub_format = self._downloader.params.get('subtitlesformat') + subtitles = [] + for sub_lang in sub_lang_list: + subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) + subtitles.append(subtitle) + return subtitles def _print_formats(self, formats): print('Available formats:') @@ -296,7 +331,7 @@ class YoutubeIE(InfoExtractor): else: raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) except (IOError, netrc.NetrcParseError) as err: - self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) + self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) return # Set language @@ -305,30 +340,65 @@ class YoutubeIE(InfoExtractor): self.report_lang() compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err)) + self._downloader.report_warning(u'unable to set language: %s' % compat_str(err)) return # No authentication to be performed if username is None: return + request = compat_urllib_request.Request(self._LOGIN_URL) + try: + login_page = compat_urllib_request.urlopen(request).read().decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err)) + return + + galx = None + dsh = None + match = re.search(re.compile(r']* name="loginForm"', login_results) is not None: - self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') + if re.search(r'(?i)]* id="gaia_loginform"', login_results) is not None: + self._downloader.report_warning(u'unable to log in: bad username or password') return except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) + self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) return # Confirm age @@ -341,13 +411,13 @@ class YoutubeIE(InfoExtractor): self.report_age_confirmation() age_results = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) + self._downloader.report_error(u'unable to confirm age: %s' % compat_str(err)) return def _extract_id(self, url): mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group(2) return video_id @@ -366,7 +436,7 @@ class YoutubeIE(InfoExtractor): try: video_webpage_bytes = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download video webpage: %s' % compat_str(err)) return video_webpage = video_webpage_bytes.decode('utf-8', 'ignore') @@ -391,18 +461,18 @@ class YoutubeIE(InfoExtractor): if 'token' in video_info: break except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download video info webpage: %s' % compat_str(err)) return if 'token' not in video_info: if 'reason' in video_info: - self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0]) + self._downloader.report_error(u'YouTube said: %s' % video_info['reason'][0]) else: - self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason') + self._downloader.report_error(u'"token" parameter not in video info for unknown reason') return # Check for "rental" videos if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info: - self._downloader.trouble(u'ERROR: "rental" videos not supported') + self._downloader.report_error(u'"rental" videos not supported') return # Start extracting information @@ -410,7 +480,7 @@ class YoutubeIE(InfoExtractor): # uploader if 'author' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract uploader name') + self._downloader.report_error(u'unable to extract uploader name') return video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0]) @@ -420,17 +490,17 @@ class YoutubeIE(InfoExtractor): if mobj is not None: video_uploader_id = mobj.group(1) else: - self._downloader.trouble(u'WARNING: unable to extract uploader nickname') + self._downloader.report_warning(u'unable to extract uploader nickname') # title if 'title' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract video title') + self._downloader.report_error(u'unable to extract video title') return video_title = compat_urllib_parse.unquote_plus(video_info['title'][0]) # thumbnail image if 'thumbnail_url' not in video_info: - self._downloader.trouble(u'WARNING: unable to extract video thumbnail') + self._downloader.report_warning(u'unable to extract video thumbnail') video_thumbnail = '' else: # don't panic if we can't find it video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0]) @@ -454,15 +524,29 @@ class YoutubeIE(InfoExtractor): else: video_description = '' - # closed captions + # subtitles video_subtitles = None + if self._downloader.params.get('writesubtitles', False): - (srt_error, video_subtitles) = self._extract_subtitles(video_id) - if srt_error: - self._downloader.trouble(srt_error) + video_subtitles = self._extract_subtitle(video_id) + if video_subtitles: + (sub_error, sub_lang, sub) = video_subtitles[0] + if sub_error: + self._downloader.trouble(sub_error) + + if self._downloader.params.get('allsubtitles', False): + video_subtitles = self._extract_all_subtitles(video_id) + for video_subtitle in video_subtitles: + (sub_error, sub_lang, sub) = video_subtitle + if sub_error: + self._downloader.trouble(sub_error) + + if self._downloader.params.get('listsubtitles', False): + sub_lang_list = self._list_available_subtitles(video_id) + return if 'length_seconds' not in video_info: - self._downloader.trouble(u'WARNING: unable to extract video duration') + self._downloader.report_warning(u'unable to extract video duration') video_duration = '' else: video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]) @@ -490,7 +574,7 @@ class YoutubeIE(InfoExtractor): format_list = available_formats existing_formats = [x for x in format_list if x in url_map] if len(existing_formats) == 0: - self._downloader.trouble(u'ERROR: no known formats available for video') + self._downloader.report_error(u'no known formats available for video') return if self._downloader.params.get('listformats', None): self._print_formats(existing_formats) @@ -511,10 +595,10 @@ class YoutubeIE(InfoExtractor): video_url_list = [(rf, url_map[rf])] break if video_url_list is None: - self._downloader.trouble(u'ERROR: requested format not available') + self._downloader.report_error(u'requested format not available') return else: - self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info') + self._downloader.report_error(u'no conn or url_encoded_fmt_stream_map information found in video info') return results = [] @@ -577,7 +661,7 @@ class MetacafeIE(InfoExtractor): self.report_disclaimer() disclaimer = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err)) + self._downloader.report_error(u'unable to retrieve disclaimer: %s' % compat_str(err)) return # Confirm age @@ -590,14 +674,14 @@ class MetacafeIE(InfoExtractor): self.report_age_confirmation() disclaimer = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) + self._downloader.report_error(u'unable to confirm age: %s' % compat_str(err)) return def _real_extract(self, url): # Extract id and simplified title from URL mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group(1) @@ -614,7 +698,7 @@ class MetacafeIE(InfoExtractor): self.report_download_webpage(video_id) webpage = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable retrieve video webpage: %s' % compat_str(err)) return # Extract URL, uploader and title from webpage @@ -634,15 +718,15 @@ class MetacafeIE(InfoExtractor): else: mobj = re.search(r' name="flashvars" value="(.*?)"', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') + self._downloader.report_error(u'unable to extract media URL') return vardict = compat_parse_qs(mobj.group(1)) if 'mediaData' not in vardict: - self._downloader.trouble(u'ERROR: unable to extract media URL') + self._downloader.report_error(u'unable to extract media URL') return mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0]) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') + self._downloader.report_error(u'unable to extract media URL') return mediaURL = mobj.group(1).replace('\\/', '/') video_extension = mediaURL[-3:] @@ -650,13 +734,13 @@ class MetacafeIE(InfoExtractor): mobj = re.search(r'(?im)(.*) - Video', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') + self._downloader.report_error(u'unable to extract title') return video_title = mobj.group(1).decode('utf-8') mobj = re.search(r'submitter=(.*?);', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') + self._downloader.report_error(u'unable to extract uploader nickname') return video_uploader = mobj.group(1) @@ -675,6 +759,7 @@ class DailymotionIE(InfoExtractor): _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)' IE_NAME = u'dailymotion' + _WORKING = False def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) @@ -687,7 +772,7 @@ class DailymotionIE(InfoExtractor): # Extract id and simplified title from URL mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group(1).split('_')[0].split('?')[0] @@ -703,7 +788,7 @@ class DailymotionIE(InfoExtractor): self.report_extraction(video_id) mobj = re.search(r'\s*var flashvars = (.*)', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') + self._downloader.report_error(u'unable to extract media URL') return flashvars = compat_urllib_parse.unquote(mobj.group(1)) @@ -713,12 +798,12 @@ class DailymotionIE(InfoExtractor): self._downloader.to_screen(u'[dailymotion] Using %s' % key) break else: - self._downloader.trouble(u'ERROR: unable to extract video URL') + self._downloader.report_error(u'unable to extract video URL') return mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video URL') + self._downloader.report_error(u'unable to extract video URL') return video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/') @@ -727,7 +812,7 @@ class DailymotionIE(InfoExtractor): mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') + self._downloader.report_error(u'unable to extract title') return video_title = unescapeHTML(mobj.group('title')) @@ -737,7 +822,7 @@ class DailymotionIE(InfoExtractor): # lookin for official user mobj_official = re.search(r'', webpage) if mobj_official is None: - self._downloader.trouble(u'WARNING: unable to extract uploader nickname') + self._downloader.report_warning(u'unable to extract uploader nickname') else: video_uploader = mobj_official.group(1) else: @@ -779,7 +864,7 @@ class PhotobucketIE(InfoExtractor): # Extract id from URL mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return video_id = mobj.group(1) @@ -792,14 +877,14 @@ class PhotobucketIE(InfoExtractor): self.report_download_webpage(video_id) webpage = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return # Extract URL, uploader, and title from webpage self.report_extraction(video_id) mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') + self._downloader.report_error(u'unable to extract media URL') return mediaURL = compat_urllib_parse.unquote(mobj.group(1)) @@ -807,7 +892,7 @@ class PhotobucketIE(InfoExtractor): mobj = re.search(r'(.*) video by (.*) - Photobucket', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') + self._downloader.report_error(u'unable to extract title') return video_title = mobj.group(1).decode('utf-8') @@ -848,7 +933,7 @@ class YahooIE(InfoExtractor): # Extract ID from URL mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return video_id = mobj.group(2) @@ -861,18 +946,18 @@ class YahooIE(InfoExtractor): try: webpage = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return mobj = re.search(r'\("id", "([0-9]+)"\);', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: Unable to extract id field') + self._downloader.report_error(u'Unable to extract id field') return yahoo_id = mobj.group(1) mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: Unable to extract vid field') + self._downloader.report_error(u'Unable to extract vid field') return yahoo_vid = mobj.group(1) @@ -885,34 +970,34 @@ class YahooIE(InfoExtractor): self.report_download_webpage(video_id) webpage = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return # Extract uploader and title from webpage self.report_extraction(video_id) mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video title') + self._downloader.report_error(u'unable to extract video title') return video_title = mobj.group(1).decode('utf-8') mobj = re.search(r'

(.*)

', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video uploader') + self._downloader.report_error(u'unable to extract video uploader') return video_uploader = mobj.group(1).decode('utf-8') # Extract video thumbnail mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') + self._downloader.report_error(u'unable to extract video thumbnail') return video_thumbnail = mobj.group(1).decode('utf-8') # Extract video description mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video description') + self._downloader.report_error(u'unable to extract video description') return video_description = mobj.group(1).decode('utf-8') if not video_description: @@ -921,13 +1006,13 @@ class YahooIE(InfoExtractor): # Extract video height and width mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video height') + self._downloader.report_error(u'unable to extract video height') return yv_video_height = mobj.group(1) mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video width') + self._downloader.report_error(u'unable to extract video width') return yv_video_width = mobj.group(1) @@ -943,13 +1028,13 @@ class YahooIE(InfoExtractor): self.report_download_webpage(video_id) webpage = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return # Extract media URL from playlist XML mobj = re.search(r'https?://)?(?:(?:www|player)\.)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)' IE_NAME = u'vimeo' def __init__(self, downloader=None): @@ -988,10 +1073,14 @@ class VimeoIE(InfoExtractor): # Extract ID from URL mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return - video_id = mobj.group(1) + video_id = mobj.group('id') + if not mobj.group('proto'): + url = 'https://' + url + if mobj.group('direct_link'): + url = 'https://vimeo.com/' + video_id # Retrieve video webpage to extract further information request = compat_urllib_request.Request(url, None, std_headers) @@ -1000,7 +1089,7 @@ class VimeoIE(InfoExtractor): webpage_bytes = compat_urllib_request.urlopen(request).read() webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return # Now we begin extracting as much information as we can from what we @@ -1013,7 +1102,7 @@ class VimeoIE(InfoExtractor): config = webpage.split(' = {config:')[1].split(',assets:')[0] config = json.loads(config) except: - self._downloader.trouble(u'ERROR: unable to extract info section') + self._downloader.report_error(u'unable to extract info section') return # Extract title @@ -1063,7 +1152,7 @@ class VimeoIE(InfoExtractor): self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality)) break else: - self._downloader.trouble(u'ERROR: no known codec found') + self._downloader.report_error(u'no known codec found') return video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \ @@ -1107,10 +1196,10 @@ class ArteTvIE(InfoExtractor): self.report_download_webpage(url) webpage = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return except ValueError as err: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return return webpage @@ -1120,7 +1209,7 @@ class ArteTvIE(InfoExtractor): info = {} if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return for (i, key, err) in matchTuples: @@ -1229,7 +1318,8 @@ class GenericIE(InfoExtractor): def report_download_webpage(self, video_id): """Report webpage download.""" - self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') + if not self._downloader.params.get('test', False): + self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id) def report_extraction(self, video_id): @@ -1282,7 +1372,7 @@ class GenericIE(InfoExtractor): opener = compat_urllib_request.OpenerDirector() for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler, HTTPMethodFallback, HEADRedirectHandler, - compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: + compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: opener.add_handler(handler()) response = opener.open(HeadRequest(url)) @@ -1299,17 +1389,12 @@ class GenericIE(InfoExtractor): if self._test_redirect(url): return video_id = url.split('/')[-1] - request = compat_urllib_request.Request(url) try: - self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) - return + webpage = self._download_webpage(url, video_id) except ValueError as err: # since this is the last-resort InfoExtractor, if # this error is thrown, it'll be thrown here - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return self.report_extraction(video_id) @@ -1319,13 +1404,16 @@ class GenericIE(InfoExtractor): # Broaden the search a little bit mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + # Broaden the search a little bit: JWPlayer JS loader + mobj = re.search(r'[^A-Za-z0-9]?file:\s*["\'](http[^\'"&]*)', webpage) + if mobj is None: + self._downloader.report_error(u'Invalid URL: %s' % url) return # It's possible that one of the regexes # matched, but returned an empty group: if mobj.group(1) is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return video_url = compat_urllib_parse.unquote(mobj.group(1)) @@ -1343,14 +1431,14 @@ class GenericIE(InfoExtractor): # and so on and so forth; it's just not practical mobj = re.search(r'(.*)', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') + self._downloader.report_error(u'unable to extract title') return video_title = mobj.group(1) # video uploader is domain name mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') + self._downloader.report_error(u'unable to extract title') return video_uploader = mobj.group(1) @@ -1382,7 +1470,7 @@ class YoutubeSearchIE(InfoExtractor): def _real_extract(self, query): mobj = re.match(self._VALID_URL, query) if mobj is None: - self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) + self._downloader.report_error(u'invalid search query "%s"' % query) return prefix, query = query.split(':') @@ -1398,10 +1486,10 @@ class YoutubeSearchIE(InfoExtractor): try: n = int(prefix) if n <= 0: - self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) + self._downloader.report_error(u'invalid download number %s for query "%s"' % (n, query)) return elif n > self._max_youtube_results: - self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) + self._downloader.report_warning(u'ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) n = self._max_youtube_results self._download_n_results(query, n) return @@ -1421,12 +1509,16 @@ class YoutubeSearchIE(InfoExtractor): result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1) request = compat_urllib_request.Request(result_url) try: - data = compat_urllib_request.urlopen(request).read() + data = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download API page: %s' % compat_str(err)) return api_response = json.loads(data)['data'] + if not 'items' in api_response: + self._downloader.trouble(u'[youtube] No video results') + return + new_ids = list(video['id'] for video in api_response['items']) video_ids += new_ids @@ -1460,7 +1552,7 @@ class GoogleSearchIE(InfoExtractor): def _real_extract(self, query): mobj = re.match(self._VALID_URL, query) if mobj is None: - self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) + self._downloader.report_error(u'invalid search query "%s"' % query) return prefix, query = query.split(':') @@ -1476,10 +1568,10 @@ class GoogleSearchIE(InfoExtractor): try: n = int(prefix) if n <= 0: - self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) + self._downloader.report_error(u'invalid download number %s for query "%s"' % (n, query)) return elif n > self._max_google_results: - self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) + self._downloader.report_warning(u'gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) n = self._max_google_results self._download_n_results(query, n) return @@ -1500,7 +1592,7 @@ class GoogleSearchIE(InfoExtractor): try: page = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return # Extract video identifiers @@ -1544,7 +1636,7 @@ class YahooSearchIE(InfoExtractor): def _real_extract(self, query): mobj = re.match(self._VALID_URL, query) if mobj is None: - self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) + self._downloader.report_error(u'invalid search query "%s"' % query) return prefix, query = query.split(':') @@ -1560,10 +1652,10 @@ class YahooSearchIE(InfoExtractor): try: n = int(prefix) if n <= 0: - self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) + self._downloader.report_error(u'invalid download number %s for query "%s"' % (n, query)) return elif n > self._max_yahoo_results: - self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) + self._downloader.report_warning(u'yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) n = self._max_yahoo_results self._download_n_results(query, n) return @@ -1585,7 +1677,7 @@ class YahooSearchIE(InfoExtractor): try: page = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return # Extract video identifiers @@ -1611,80 +1703,94 @@ class YahooSearchIE(InfoExtractor): class YoutubePlaylistIE(InfoExtractor): """Information Extractor for YouTube playlists.""" - _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*' - _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en' - _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s' - _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" + _VALID_URL = r"""(?: + (?:https?://)? + (?:\w+\.)? + youtube\.com/ + (?: + (?:course|view_play_list|my_playlists|artist|playlist|watch) + \? (?:.*?&)*? (?:p|a|list)= + | user/.*?/user/ + | p/ + | user/.*?#[pg]/c/ + ) + ((?:PL|EC|UU)?[0-9A-Za-z-_]{10,}) + .* + | + ((?:PL|EC|UU)[0-9A-Za-z-_]{10,}) + )""" + _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/playlists/%s?max-results=%i&start-index=%i&v=2&alt=json' + _MAX_RESULTS = 50 IE_NAME = u'youtube:playlist' def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) + @classmethod + def suitable(cls, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(cls._VALID_URL, url, re.VERBOSE) is not None + def report_download_page(self, playlist_id, pagenum): """Report attempt to download playlist page with given number.""" self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum)) def _real_extract(self, url): # Extract playlist id - mobj = re.match(self._VALID_URL, url) + mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: - self._downloader.trouble(u'ERROR: invalid url: %s' % url) - return - - # Single video case - if mobj.group(3) is not None: - self._downloader.download([mobj.group(3)]) + self._downloader.report_error(u'invalid url: %s' % url) return - # Download playlist pages - # prefix is 'p' as default for playlists but there are other types that need extra care - playlist_prefix = mobj.group(1) - if playlist_prefix == 'a': - playlist_access = 'artist' - else: - playlist_prefix = 'p' - playlist_access = 'view_play_list' - playlist_id = mobj.group(2) - video_ids = [] - pagenum = 1 + # Download playlist videos from API + playlist_id = mobj.group(1) or mobj.group(2) + page_num = 1 + videos = [] while True: - self.report_download_page(playlist_id, pagenum) - url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum) - request = compat_urllib_request.Request(url) + self.report_download_page(playlist_id, page_num) + + url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, self._MAX_RESULTS * (page_num - 1) + 1) try: - page = compat_urllib_request.urlopen(request).read().decode('utf-8') + page = compat_urllib_request.urlopen(url).read().decode('utf8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return - # Extract video identifiers - ids_in_page = [] - for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page): - if mobj.group(1) not in ids_in_page: - ids_in_page.append(mobj.group(1)) - video_ids.extend(ids_in_page) + try: + response = json.loads(page) + except ValueError as err: + self._downloader.report_error(u'Invalid JSON in API response: ' + compat_str(err)) + return - if self._MORE_PAGES_INDICATOR not in page: + if not 'feed' in response or not 'entry' in response['feed']: + self._downloader.report_error(u'Got a malformed response from YouTube API') + return + videos += [ (entry['yt$position']['$t'], entry['content']['src']) + for entry in response['feed']['entry'] + if 'content' in entry ] + + if len(response['feed']['entry']) < self._MAX_RESULTS: break - pagenum = pagenum + 1 + page_num += 1 - total = len(video_ids) + videos = [v[1] for v in sorted(videos)] + total = len(videos) playliststart = self._downloader.params.get('playliststart', 1) - 1 playlistend = self._downloader.params.get('playlistend', -1) if playlistend == -1: - video_ids = video_ids[playliststart:] + videos = videos[playliststart:] else: - video_ids = video_ids[playliststart:playlistend] + videos = videos[playliststart:playlistend] - if len(video_ids) == total: + if len(videos) == total: self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total)) else: - self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids))) + self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) - for id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) + for video in videos: + self._downloader.download([video]) return @@ -1704,7 +1810,7 @@ class YoutubeChannelIE(InfoExtractor): # Extract channel id mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid url: %s' % url) + self._downloader.report_error(u'invalid url: %s' % url) return # Download channel pages @@ -1719,7 +1825,7 @@ class YoutubeChannelIE(InfoExtractor): try: page = compat_urllib_request.urlopen(request).read().decode('utf8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return # Extract video identifiers @@ -1762,7 +1868,7 @@ class YoutubeUserIE(InfoExtractor): # Extract username mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid url: %s' % url) + self._downloader.report_error(u'invalid url: %s' % url) return username = mobj.group(1) @@ -1784,7 +1890,7 @@ class YoutubeUserIE(InfoExtractor): try: page = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return # Extract video identifiers @@ -1842,7 +1948,7 @@ class BlipTVUserIE(InfoExtractor): # Extract username mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid url: %s' % url) + self._downloader.report_error(u'invalid url: %s' % url) return username = mobj.group(1) @@ -1856,7 +1962,7 @@ class BlipTVUserIE(InfoExtractor): mobj = re.search(r'data-users-id="([^"]+)"', page) page_base = page_base % mobj.group(1) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return @@ -1870,13 +1976,12 @@ class BlipTVUserIE(InfoExtractor): while True: self.report_download_page(username, pagenum) - - request = compat_urllib_request.Request( page_base + "&page=" + str(pagenum) ) - + url = page_base + "&page=" + str(pagenum) + request = compat_urllib_request.Request( url ) try: page = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % str(err)) return # Extract video identifiers @@ -1940,7 +2045,7 @@ class DepositFilesIE(InfoExtractor): self.report_download_webpage(file_id) webpage = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve file webpage: %s' % compat_str(err)) return # Search for the real file URL @@ -1950,9 +2055,9 @@ class DepositFilesIE(InfoExtractor): mobj = re.search(r'(Attention.*?)', webpage, re.DOTALL) if (mobj is not None) and (mobj.group(1) is not None): restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip() - self._downloader.trouble(u'ERROR: %s' % restriction_message) + self._downloader.report_error(u'%s' % restriction_message) else: - self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url) + self._downloader.report_error(u'unable to extract download URL from: %s' % url) return file_url = mobj.group(1) @@ -1961,7 +2066,7 @@ class DepositFilesIE(InfoExtractor): # Search for file title mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') + self._downloader.report_error(u'unable to extract title') return file_title = mobj.group(1).decode('utf-8') @@ -1978,62 +2083,14 @@ class DepositFilesIE(InfoExtractor): class FacebookIE(InfoExtractor): """Information Extractor for Facebook""" - _WORKING = False _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P\d+)(?:.*)' _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&' _NETRC_MACHINE = 'facebook' - _available_formats = ['video', 'highqual', 'lowqual'] - _video_extensions = { - 'video': 'mp4', - 'highqual': 'mp4', - 'lowqual': 'mp4', - } IE_NAME = u'facebook' - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def _reporter(self, message): - """Add header and report message.""" - self._downloader.to_screen(u'[facebook] %s' % message) - def report_login(self): """Report attempt to log in.""" - self._reporter(u'Logging in') - - def report_video_webpage_download(self, video_id): - """Report attempt to download video webpage.""" - self._reporter(u'%s: Downloading video webpage' % video_id) - - def report_information_extraction(self, video_id): - """Report attempt to extract video information.""" - self._reporter(u'%s: Extracting video information' % video_id) - - def _parse_page(self, video_webpage): - """Extract video information from page""" - # General data - data = {'title': r'\("video_title", "(.*?)"\)', - 'description': r'
(.*?)
', - 'owner': r'\("video_owner_name", "(.*?)"\)', - 'thumbnail': r'\("thumb_url", "(?P.*?)"\)', - } - video_info = {} - for piece in data.keys(): - mobj = re.search(data[piece], video_webpage) - if mobj is not None: - video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) - - # Video urls - video_urls = {} - for fmt in self._available_formats: - mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage) - if mobj is not None: - # URL is in a Javascript segment inside an escaped Unicode format within - # the generally utf-8 page - video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) - video_info['video_urls'] = video_urls - - return video_info + self._downloader.to_screen(u'[%s] Logging in' % self.IE_NAME) def _real_initialize(self): if self._downloader is None: @@ -2056,7 +2113,7 @@ class FacebookIE(InfoExtractor): else: raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) except (IOError, netrc.NetrcParseError) as err: - self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) + self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) return if useremail is None: @@ -2073,113 +2130,52 @@ class FacebookIE(InfoExtractor): self.report_login() login_results = compat_urllib_request.urlopen(request).read() if re.search(r'', login_results) is not None: - self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') + self._downloader.report_warning(u'unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') return except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) + self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) return def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group('ID') - # Get video webpage - self.report_video_webpage_download(video_id) - request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id) - try: - page = compat_urllib_request.urlopen(request) - video_webpage = page.read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - return - - # Start extracting information - self.report_information_extraction(video_id) - - # Extract information - video_info = self._parse_page(video_webpage) - - # uploader - if 'owner' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') - return - video_uploader = video_info['owner'] - - # title - if 'title' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = video_info['title'] - video_title = video_title.decode('utf-8') - - # thumbnail image - if 'thumbnail' not in video_info: - self._downloader.trouble(u'WARNING: unable to extract video thumbnail') - video_thumbnail = '' - else: - video_thumbnail = video_info['thumbnail'] - - # upload date - upload_date = None - if 'upload_date' in video_info: - upload_time = video_info['upload_date'] - timetuple = email.utils.parsedate_tz(upload_time) - if timetuple is not None: - try: - upload_date = time.strftime('%Y%m%d', timetuple[0:9]) - except: - pass - - # description - video_description = video_info.get('description', 'No description available.') - - url_map = video_info['video_urls'] - if url_map: - # Decide which formats to download - req_format = self._downloader.params.get('format', None) - format_limit = self._downloader.params.get('format_limit', None) + url = 'https://www.facebook.com/video/video.php?v=%s' % video_id + webpage = self._download_webpage(url, video_id) - if format_limit is not None and format_limit in self._available_formats: - format_list = self._available_formats[self._available_formats.index(format_limit):] - else: - format_list = self._available_formats - existing_formats = [x for x in format_list if x in url_map] - if len(existing_formats) == 0: - self._downloader.trouble(u'ERROR: no known formats available for video') - return - if req_format is None: - video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality - elif req_format == 'worst': - video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality - elif req_format == '-1': - video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats - else: - # Specific format - if req_format not in url_map: - self._downloader.trouble(u'ERROR: requested format not available') - return - video_url_list = [(req_format, url_map[req_format])] # Specific format + BEFORE = '[["allowFullScreen","true"],["allowScriptAccess","always"],["salign","tl"],["scale","noscale"],["wmode","opaque"]].forEach(function(param) {swf.addParam(param[0], param[1]);});\n' + AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});' + m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage) + if not m: + raise ExtractorError(u'Cannot parse data') + data = dict(json.loads(m.group(1))) + params_raw = compat_urllib_parse.unquote(data['params']) + params = json.loads(params_raw) + video_url = params['hd_src'] + if not video_url: + video_url = params['sd_src'] + if not video_url: + raise ExtractorError(u'Cannot find video URL') + video_duration = int(params['video_duration']) + + m = re.search('

([^<]+)

', webpage) + if not m: + raise ExtractorError(u'Cannot find title in webpage') + video_title = unescapeHTML(m.group(1)) - results = [] - for format_param, video_real_url in video_url_list: - # Extension - video_extension = self._video_extensions.get(format_param, 'mp4') + info = { + 'id': video_id, + 'title': video_title, + 'url': video_url, + 'ext': 'mp4', + 'duration': video_duration, + 'thumbnail': params['thumbnail_src'], + } + return [info] - results.append({ - 'id': video_id.decode('utf-8'), - 'url': video_real_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), - 'upload_date': upload_date, - 'title': video_title, - 'ext': video_extension.decode('utf-8'), - 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), - 'thumbnail': video_thumbnail.decode('utf-8'), - 'description': video_description.decode('utf-8'), - }) - return results class BlipTVIE(InfoExtractor): """Information extractor for blip.tv""" @@ -2199,15 +2195,27 @@ class BlipTVIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return + urlp = compat_urllib_parse_urlparse(url) + if urlp.path.startswith('/play/'): + request = compat_urllib_request.Request(url) + response = compat_urllib_request.urlopen(request) + redirecturl = response.geturl() + rurlp = compat_urllib_parse_urlparse(redirecturl) + file_id = compat_parse_qs(rurlp.fragment)['file'][0].rpartition('/')[2] + url = 'http://blip.tv/a/a-' + file_id + return self._real_extract(url) + + if '?' in url: cchar = '&' else: cchar = '?' json_url = url + cchar + 'skin=json&version=2&no_wrap=1' request = compat_urllib_request.Request(json_url) + request.add_header('User-Agent', 'iTunes/10.6.1') self.report_extraction(mobj.group(1)) info = None try: @@ -2228,14 +2236,13 @@ class BlipTVIE(InfoExtractor): 'urlhandle': urlh } except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) - return + raise ExtractorError(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) if info is None: # Regular URL try: json_code_bytes = urlh.read() json_code = json_code_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to read video info webpage: %s' % compat_str(err)) return try: @@ -2262,13 +2269,13 @@ class BlipTVIE(InfoExtractor): 'format': data['media']['mimeType'], 'thumbnail': data['thumbnailUrl'], 'description': data['description'], - 'player_url': data['embedUrl'] + 'player_url': data['embedUrl'], + 'user_agent': 'iTunes/10.6.1', } except (ValueError,KeyError) as err: - self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err)) + self._downloader.report_error(u'unable to parse video information: %s' % repr(err)) return - std_headers['User-Agent'] = 'iTunes/10.6.1' return [info] @@ -2288,7 +2295,7 @@ class MyVideoIE(InfoExtractor): def _real_extract(self,url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._download.trouble(u'ERROR: invalid URL: %s' % url) + self._download.report_error(u'invalid URL: %s' % url) return video_id = mobj.group(1) @@ -2298,16 +2305,16 @@ class MyVideoIE(InfoExtractor): webpage = self._download_webpage(webpage_url, video_id) self.report_extraction(video_id) - mobj = re.search(r'', + mobj = re.search(r'([^<]+)', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') + self._downloader.report_error(u'unable to extract title') return video_title = mobj.group(1) @@ -2337,7 +2344,6 @@ class ComedyCentralIE(InfoExtractor): (the-colbert-report-(videos|collections)/(?P[0-9]+)/[^/]*/(?P.*?)) |(watch/(?P[^/]*)/(?P.*))))) $""" - IE_NAME = u'comedycentral' _available_formats = ['3500', '2200', '1700', '1200', '750', '400'] @@ -2358,23 +2364,20 @@ class ComedyCentralIE(InfoExtractor): '400': '384x216', } - def suitable(self, url): + @classmethod + def suitable(cls, url): """Receives a URL and returns True if suitable for this IE.""" - return re.match(self._VALID_URL, url, re.VERBOSE) is not None + return re.match(cls._VALID_URL, url, re.VERBOSE) is not None def report_extraction(self, episode_id): self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) - def report_config_download(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id) + def report_config_download(self, episode_id, media_id): + self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration for %s' % (episode_id, media_id)) def report_index_download(self, episode_id): self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id) - def report_player_url(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id) - - def _print_formats(self, formats): print('Available formats:') for x in formats: @@ -2384,7 +2387,7 @@ class ComedyCentralIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return if mobj.group('shortname'): @@ -2413,57 +2416,49 @@ class ComedyCentralIE(InfoExtractor): try: htmlHandle = compat_urllib_request.urlopen(req) html = htmlHandle.read() + webpage = html.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return if dlNewest: url = htmlHandle.geturl() mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: - self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url) + self._downloader.report_error(u'Invalid redirected URL: ' + url) return if mobj.group('episode') == '': - self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url) + self._downloader.report_error(u'Redirected URL is still not specific: ' + url) return epTitle = mobj.group('episode') - mMovieParams = re.findall('(?:gsp.comedystor/.*)$', rtmp_video_url) + if not m: + raise ExtractorError(u'Cannot transform RTMP url') + base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/' + video_url = base + m.group('finalid') - if video_url.startswith(broken_cdn): - video_url = video_url.replace(broken_cdn, better_cdn) - - effTitle = showId + u'-' + epTitle + effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1) info = { 'id': shortMediaId, 'url': video_url, @@ -2525,9 +2518,7 @@ class ComedyCentralIE(InfoExtractor): 'format': format, 'thumbnail': None, 'description': officialTitle, - 'player_url': None #playerUrl } - results.append(info) return results @@ -2548,7 +2539,7 @@ class EscapistIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return showName = mobj.group('showname') videoId = mobj.group('episode') @@ -2560,7 +2551,7 @@ class EscapistIE(InfoExtractor): m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type']) webPage = webPageBytes.decode(m.group(1) if m else 'utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err)) + self._downloader.report_error(u'unable to download webpage: ' + compat_str(err)) return descMatch = re.search('(.*?)\s+-\s+XVID', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video title') + self._downloader.report_error(u'unable to extract video title') return video_title = mobj.group(1) @@ -2721,7 +2711,7 @@ class XVideosIE(InfoExtractor): # Extract video thumbnail mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') + self._downloader.report_error(u'unable to extract video thumbnail') return video_thumbnail = mobj.group(0) @@ -2765,7 +2755,7 @@ class SoundcloudIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return # extract uploader (which is in the url) @@ -2783,7 +2773,7 @@ class SoundcloudIE(InfoExtractor): info_json_bytes = compat_urllib_request.urlopen(request).read() info_json = info_json_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download video webpage: %s' % compat_str(err)) return info = json.loads(info_json) @@ -2796,7 +2786,7 @@ class SoundcloudIE(InfoExtractor): stream_json_bytes = compat_urllib_request.urlopen(request).read() stream_json = stream_json_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download stream definitions: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download stream definitions: %s' % compat_str(err)) return streams = json.loads(stream_json) @@ -2824,7 +2814,7 @@ class InfoQIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return webpage = self._download_webpage(url, video_id=url) @@ -2833,7 +2823,7 @@ class InfoQIE(InfoExtractor): # Extract video URL mobj = re.search(r"jsclassref='([^']*)'", webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video url') + self._downloader.report_error(u'unable to extract video url') return real_id = compat_urllib_parse.unquote(base64.b64decode(mobj.group(1).encode('ascii')).decode('utf-8')) video_url = 'rtmpe://video.infoq.com/cfx/st/' + real_id @@ -2841,7 +2831,7 @@ class InfoQIE(InfoExtractor): # Extract title mobj = re.search(r'contentTitle = "(.*?)";', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video title') + self._downloader.report_error(u'unable to extract video title') return video_title = mobj.group(1) @@ -2924,7 +2914,7 @@ class MixcloudIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return # extract uploader & filename from url uploader = mobj.group(1).decode('utf-8') @@ -2938,7 +2928,7 @@ class MixcloudIE(InfoExtractor): self.report_download_json(file_url) jsonData = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve file: %s' % compat_str(err)) return # parse JSON @@ -2962,7 +2952,7 @@ class MixcloudIE(InfoExtractor): break # got it! else: if req_format not in formats: - self._downloader.trouble(u'ERROR: format is not available') + self._downloader.report_error(u'format is not available') return url_list = self.get_urls(formats, req_format) @@ -2999,8 +2989,7 @@ class StanfordOpenClassroomIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return + raise ExtractorError(u'Invalid URL: %s' % url) if mobj.group('course') and mobj.group('video'): # A specific video course = mobj.group('course') @@ -3017,14 +3006,14 @@ class StanfordOpenClassroomIE(InfoExtractor): try: metaXml = compat_urllib_request.urlopen(xmlUrl).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download video info XML: %s' % compat_str(err)) return mdoc = xml.etree.ElementTree.fromstring(metaXml) try: info['title'] = mdoc.findall('./title')[0].text info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text except IndexError: - self._downloader.trouble(u'\nERROR: Invalid metadata XML file') + self._downloader.report_error(u'Invalid metadata XML file') return info['ext'] = info['url'].rpartition('.')[2] return [info] @@ -3037,12 +3026,9 @@ class StanfordOpenClassroomIE(InfoExtractor): 'upload_date': None, } - self.report_download_webpage(info['id']) - try: - coursepage = compat_urllib_request.urlopen(url).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err)) - return + coursepage = self._download_webpage(url, info['id'], + note='Downloading course info page', + errnote='Unable to download course info page') m = re.search('

([^<]+)

', coursepage) if m: @@ -3066,7 +3052,6 @@ class StanfordOpenClassroomIE(InfoExtractor): assert entry['type'] == 'reference' results += self.extract(entry['url']) return results - else: # Root page info = { 'id': 'Stanford OpenClassroom', @@ -3080,7 +3065,7 @@ class StanfordOpenClassroomIE(InfoExtractor): try: rootpage = compat_urllib_request.urlopen(rootURL).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err)) + self._downloader.report_error(u'unable to download course info page: ' + compat_str(err)) return info['title'] = info['id'] @@ -3112,7 +3097,7 @@ class MTVIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return if not mobj.group('proto'): url = 'http://' + url @@ -3122,25 +3107,25 @@ class MTVIE(InfoExtractor): mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract song name') + self._downloader.report_error(u'unable to extract song name') return song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1')) mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract performer') + self._downloader.report_error(u'unable to extract performer') return performer = unescapeHTML(mobj.group(1).decode('iso-8859-1')) video_title = performer + ' - ' + song_name mobj = re.search(r'', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to mtvn_uri') + self._downloader.report_error(u'unable to mtvn_uri') return mtvn_uri = mobj.group(1) mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract content id') + self._downloader.report_error(u'unable to extract content id') return content_id = mobj.group(1) @@ -3150,7 +3135,7 @@ class MTVIE(InfoExtractor): try: metadataXml = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download video metadata: %s' % compat_str(err)) return mdoc = xml.etree.ElementTree.fromstring(metadataXml) @@ -3222,7 +3207,7 @@ class YoukuIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group('ID') @@ -3233,7 +3218,7 @@ class YoukuIE(InfoExtractor): self.report_download_webpage(video_id) jsondata = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return self.report_extraction(video_id) @@ -3264,7 +3249,7 @@ class YoukuIE(InfoExtractor): fileid = config['data'][0]['streamfileids'][format] keys = [s['k'] for s in config['data'][0]['segs'][format]] except (UnicodeDecodeError, ValueError, KeyError): - self._downloader.trouble(u'ERROR: unable to extract info section') + self._downloader.report_error(u'unable to extract info section') return files_info=[] @@ -3294,7 +3279,7 @@ class YoukuIE(InfoExtractor): class XNXXIE(InfoExtractor): """Information extractor for xnxx.com""" - _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)' + _VALID_URL = r'^(?:https?://)?video\.xnxx\.com/video([0-9]+)/(.*)' IE_NAME = u'xnxx' VIDEO_URL_RE = r'flv_url=(.*?)&' VIDEO_TITLE_RE = r'(.*?)\s+-\s+XNXX.COM' @@ -3311,7 +3296,7 @@ class XNXXIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group(1) @@ -3322,24 +3307,24 @@ class XNXXIE(InfoExtractor): webpage_bytes = compat_urllib_request.urlopen(url).read() webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err) + self._downloader.report_error(u'unable to download video webpage: %s' % err) return result = re.search(self.VIDEO_URL_RE, webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video url') + self._downloader.report_error(u'unable to extract video url') return video_url = compat_urllib_parse.unquote(result.group(1)) result = re.search(self.VIDEO_TITLE_RE, webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video title') + self._downloader.report_error(u'unable to extract video title') return video_title = result.group(1) result = re.search(self.VIDEO_THUMB_RE, webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') + self._downloader.report_error(u'unable to extract video thumbnail') return video_thumbnail = result.group(1) @@ -3388,7 +3373,7 @@ class GooglePlusIE(InfoExtractor): # Extract id from URL mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + self._downloader.report_error(u'Invalid URL: %s' % url) return post_url = mobj.group(0) @@ -3402,7 +3387,7 @@ class GooglePlusIE(InfoExtractor): try: webpage = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve entry webpage: %s' % compat_str(err)) return # Extract update date @@ -3437,14 +3422,14 @@ class GooglePlusIE(InfoExtractor): pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]' mobj = re.search(pattern, webpage) if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video page URL') + self._downloader.report_error(u'unable to extract video page URL') video_page = mobj.group(1) request = compat_urllib_request.Request(video_page) try: webpage = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err)) return self.report_extract_vid_page(video_page) @@ -3454,7 +3439,7 @@ class GooglePlusIE(InfoExtractor): pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"' mobj = re.findall(pattern, webpage) if len(mobj) == 0: - self._downloader.trouble(u'ERROR: unable to extract video links') + self._downloader.report_error(u'unable to extract video links') # Sort in resolution links = sorted(mobj) @@ -3486,7 +3471,7 @@ class NBAIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group(1) @@ -3542,21 +3527,29 @@ class JustinTVIE(InfoExtractor): webpage_bytes = urlh.read() webpage = webpage_bytes.decode('utf-8', 'ignore') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video info JSON: %s' % compat_str(err)) + self._downloader.report_error(u'unable to download video info JSON: %s' % compat_str(err)) return response = json.loads(webpage) + if type(response) != list: + error_text = response.get('error', 'unknown error') + self._downloader.report_error(u'Justin.tv API: %s' % error_text) + return info = [] for clip in response: video_url = clip['video_file_url'] if video_url: video_extension = os.path.splitext(video_url)[1][1:] - video_date = re.sub('-', '', clip['created_on'][:10]) + video_date = re.sub('-', '', clip['start_time'][:10]) + video_uploader_id = clip.get('user_id', clip.get('channel_id')) + video_id = clip['id'] + video_title = clip.get('title', video_id) info.append({ - 'id': clip['id'], + 'id': video_id, 'url': video_url, - 'title': clip['title'], - 'uploader': clip.get('user_id', clip.get('channel_id')), + 'title': video_title, + 'uploader': clip.get('channel_name', video_uploader_id), + 'uploader_id': video_uploader_id, 'upload_date': video_date, 'ext': video_extension, }) @@ -3565,7 +3558,7 @@ class JustinTVIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return api = 'http://api.justin.tv' @@ -3575,7 +3568,7 @@ class JustinTVIE(InfoExtractor): paged = True api += '/channel/archives/%s.json' else: - api += '/clip/show/%s.json' + api += '/broadcast/by_archive/%s.json' api = api % (video_id,) self.report_extraction(video_id) @@ -3600,7 +3593,7 @@ class FunnyOrDieIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return video_id = mobj.group('id') @@ -3608,13 +3601,13 @@ class FunnyOrDieIE(InfoExtractor): m = re.search(r'<video[^>]*>\s*<source[^>]*>\s*<source src="(?P<url>[^"]+)"', webpage, re.DOTALL) if not m: - self._downloader.trouble(u'ERROR: unable to find video information') + self._downloader.report_error(u'unable to find video information') video_url = unescapeHTML(m.group('url')) - m = re.search(r"class='player_page_h1'>\s+<a.*?>(?P<title>.*?)</a>", webpage) + m = re.search(r"<h1 class='player_page_h1'.*?>(?P<title>.*?)</h1>", webpage, flags=re.DOTALL) if not m: self._downloader.trouble(u'Cannot find video title') - title = unescapeHTML(m.group('title')) + title = clean_html(m.group('title')) m = re.search(r'<meta property="og:description" content="(?P<desc>.*?)"', webpage) if m: @@ -3631,65 +3624,17 @@ class FunnyOrDieIE(InfoExtractor): } return [info] -class TweetReelIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P<id>[0-9a-z]+)$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - video_id = mobj.group('id') - webpage = self._download_webpage(url, video_id) - - m = re.search(r'<div id="left" status_id="([0-9]+)">', webpage) - if not m: - self._downloader.trouble(u'ERROR: Cannot find status ID') - status_id = m.group(1) - - m = re.search(r'<div class="tweet_text">(.*?)</div>', webpage, flags=re.DOTALL) - if not m: - self._downloader.trouble(u'WARNING: Cannot find description') - desc = unescapeHTML(re.sub('<a.*?</a>', '', m.group(1))).strip() - - m = re.search(r'<div class="tweet_info">.*?from <a target="_blank" href="https?://twitter.com/(?P<uploader_id>.+?)">(?P<uploader>.+?)</a>', webpage, flags=re.DOTALL) - if not m: - self._downloader.trouble(u'ERROR: Cannot find uploader') - uploader = unescapeHTML(m.group('uploader')) - uploader_id = unescapeHTML(m.group('uploader_id')) - - m = re.search(r'<span unixtime="([0-9]+)"', webpage) - if not m: - self._downloader.trouble(u'ERROR: Cannot find upload date') - upload_date = datetime.datetime.fromtimestamp(int(m.group(1))).strftime('%Y%m%d') - - title = desc - video_url = 'http://files.tweetreel.com/video/' + status_id + '.mov' - - info = { - 'id': video_id, - 'url': video_url, - 'ext': 'mov', - 'title': title, - 'description': desc, - 'uploader': uploader, - 'uploader_id': uploader_id, - 'internal_id': status_id, - 'upload_date': upload_date - } - return [info] - class SteamIE(InfoExtractor): - _VALID_URL = r"""http://store.steampowered.com/ + _VALID_URL = r"""http://store.steampowered.com/ (?P<urltype>video|app)/ #If the page is only for videos or for a game (?P<gameID>\d+)/? (?P<videoID>\d*)(?P<extra>\??) #For urltype == video we sometimes get the videoID """ - def suitable(self, url): + @classmethod + def suitable(cls, url): """Receives a URL and returns True if suitable for this IE.""" - return re.match(self._VALID_URL, url, re.VERBOSE) is not None + return re.match(cls._VALID_URL, url, re.VERBOSE) is not None def _real_extract(self, url): m = re.match(self._VALID_URL, url, re.VERBOSE) @@ -3700,26 +3645,30 @@ class SteamIE(InfoExtractor): mweb = re.finditer(urlRE, webpage) namesRE = r'<span class="title">(?P<videoName>.+?)</span>' titles = re.finditer(namesRE, webpage) + thumbsRE = r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">' + thumbs = re.finditer(thumbsRE, webpage) videos = [] - for vid,vtitle in zip(mweb,titles): + for vid,vtitle,thumb in zip(mweb,titles,thumbs): video_id = vid.group('videoID') title = vtitle.group('videoName') video_url = vid.group('videoURL') + video_thumb = thumb.group('thumbnail') if not video_url: - self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id) + self._downloader.report_error(u'Cannot find video url for %s' % video_id) info = { 'id':video_id, 'url':video_url, 'ext': 'flv', - 'title': unescapeHTML(title) + 'title': unescapeHTML(title), + 'thumbnail': video_thumb } videos.append(info) return videos - + class UstreamIE(InfoExtractor): - _VALID_URL = r'http://www.ustream.tv/recorded/(?P<videoID>\d+)' + _VALID_URL = r'https?://www\.ustream\.tv/recorded/(?P<videoID>\d+)' IE_NAME = u'ustream' - + def _real_extract(self, url): m = re.match(self._VALID_URL, url) video_id = m.group('videoID') @@ -3738,49 +3687,109 @@ class UstreamIE(InfoExtractor): } return [info] +class WorldStarHipHopIE(InfoExtractor): + _VALID_URL = r'http://(?:www|m)\.worldstar(?:candy|hiphop)\.com/videos/video\.php\?v=(?P<id>.*)' + IE_NAME = u'WorldStarHipHop' + def _real_extract(self, url): + _src_url = r"""(http://hw-videos.*(?:mp4|flv))""" -class YouPornIE(InfoExtractor): - """Information extractor for youporn.com.""" + webpage_src = compat_urllib_request.urlopen(url).read() + webpage_src = webpage_src.decode('utf-8') - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+)' - IE_NAME = u'youporn' - VIDEO_TITLE_RE = r'videoTitleArea">(?P<title>.*)</h1>' - VIDEO_DATE_RE = r'Date:</b>(?P<date>.*)</li>' - VIDEO_UPLOADER_RE = r'Submitted:</b>(?P<uploader>.*)</li>' - DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>' - LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">' + mobj = re.search(_src_url, webpage_src) - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) + m = re.match(self._VALID_URL, url) + video_id = m.group('id') - def report_id(self, video_id): - """Report finding video ID""" - self._downloader.to_screen(u'[youporn] Video ID: %s' % video_id) + if mobj is not None: + video_url = mobj.group() + if 'mp4' in video_url: + ext = 'mp4' + else: + ext = 'flv' + else: + self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id) + return - def report_webpage(self, url): - """Report downloading page""" - self._downloader.to_screen(u'[youporn] Downloaded page: %s' % url) + _title = r"""<title>(.*)""" - def report_title(self, video_title): - """Report dfinding title""" - self._downloader.to_screen(u'[youporn] Title: %s' % video_title) - - def report_uploader(self, uploader): - """Report dfinding title""" - self._downloader.to_screen(u'[youporn] Uploader: %s' % uploader) + mobj = re.search(_title, webpage_src) + + if mobj is not None: + title = mobj.group(1) + else: + title = 'World Start Hip Hop - %s' % time.ctime() + + _thumbnail = r"""rel="image_src" href="(.*)" />""" + mobj = re.search(_thumbnail, webpage_src) - def report_upload_date(self, video_date): - """Report finding date""" - self._downloader.to_screen(u'[youporn] Date: %s' % video_date) + # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video. + if mobj is not None: + thumbnail = mobj.group(1) + else: + _title = r"""candytitles.*>(.*)""" + mobj = re.search(_title, webpage_src) + if mobj is not None: + title = mobj.group(1) + thumbnail = None + + results = [{ + 'id': video_id, + 'url' : video_url, + 'title' : title, + 'thumbnail' : thumbnail, + 'ext' : ext, + }] + return results + +class RBMARadioIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P[^/]+)$' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('videoID') + + webpage = self._download_webpage(url, video_id) + m = re.search(r'', webpage) + if not m: + raise ExtractorError(u'Cannot find metadata') + json_data = m.group(1) + + try: + data = json.loads(json_data) + except ValueError as e: + raise ExtractorError(u'Invalid JSON: ' + str(e)) + + video_url = data['akamai_url'] + '&cbr=256' + url_parts = compat_urllib_parse_urlparse(video_url) + video_ext = url_parts.path.rpartition('.')[2] + info = { + 'id': video_id, + 'url': video_url, + 'ext': video_ext, + 'title': data['title'], + 'description': data.get('teaser_text'), + 'location': data.get('country_of_origin'), + 'uploader': data.get('host', {}).get('name'), + 'uploader_id': data.get('host', {}).get('slug'), + 'thumbnail': data.get('image', {}).get('large_url_2x'), + 'duration': data.get('duration'), + } + return [info] + + +class YouPornIE(InfoExtractor): + """Information extractor for youporn.com.""" + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P[0-9]+)/(?P[^/]+)' def _print_formats(self, formats): """Print all available formats""" - print 'Available formats:' - print u'ext\t\tformat' - print u'---------------------------------' + print(u'Available formats:') + print(u'ext\t\tformat') + print(u'---------------------------------') for format in formats: - print u'%s\t\t%s' % (format['ext'], format['format']) + print(u'%s\t\t%s' % (format['ext'], format['format'])) def _specific(self, req_format, formats): for x in formats: @@ -3788,63 +3797,55 @@ class YouPornIE(InfoExtractor): return x return None - def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return - video_id = mobj.group('videoid').decode('utf-8') - self.report_id(video_id) + video_id = mobj.group('videoid') - # Get webpage content - try: - webpage = urllib2.urlopen(url).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err) - return - self.report_webpage(url) + req = compat_urllib_request.Request(url) + req.add_header('Cookie', 'age_verified=1') + webpage = self._download_webpage(req, video_id) # Get the video title - result = re.search(self.VIDEO_TITLE_RE, webpage) + result = re.search(r'<h1.*?>(?P<title>.*)</h1>', webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = result.group('title').decode('utf-8').strip() - self.report_title(video_title) + raise ExtractorError(u'Unable to extract video title') + video_title = result.group('title').strip() # Get the video date - result = re.search(self.VIDEO_DATE_RE, webpage) + result = re.search(r'Date:</label>(?P<date>.*) </li>', webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video date') - return - upload_date = result.group('date').decode('utf-8').strip() - self.report_upload_date(upload_date) + self._downloader.report_warning(u'unable to extract video date') + upload_date = None + else: + upload_date = result.group('date').strip() # Get the video uploader - result = re.search(self.VIDEO_UPLOADER_RE, webpage) + result = re.search(r'Submitted:</label>(?P<uploader>.*)</li>', webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract uploader') - return - video_uploader = result.group('uploader').decode('utf-8').strip() - video_uploader = clean_html( video_uploader ) - self.report_uploader(video_uploader) + self._downloader.report_warning(u'unable to extract uploader') + video_uploader = None + else: + video_uploader = result.group('uploader').strip() + video_uploader = clean_html( video_uploader ) # Get all of the formats available - result = re.search(self.DOWNLOAD_LIST_RE, webpage) + DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>' + result = re.search(DOWNLOAD_LIST_RE, webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract download list') - return - download_list_html = result.group('download_list').decode('utf-8').strip() + raise ExtractorError(u'Unable to extract download list') + download_list_html = result.group('download_list').strip() # Get all of the links from the page - links = re.findall(self.LINK_RE, download_list_html) + LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">' + links = re.findall(LINK_RE, download_list_html) if(len(links) == 0): - self._downloader.trouble(u'ERROR: no known formats available for video') - return - - self._downloader.to_screen(u'[youporn] Links found: %d' % len(links)) + raise ExtractorError(u'ERROR: no known formats available for video') + + self._downloader.to_screen(u'[youporn] Links found: %d' % len(links)) formats = [] for link in links: @@ -3853,8 +3854,8 @@ class YouPornIE(InfoExtractor): # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0 # A path looks like this: # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4 - video_url = unescapeHTML( link.decode('utf-8') ) - path = urlparse( video_url ).path + video_url = unescapeHTML( link ) + path = compat_urllib_parse_urlparse( video_url ).path extension = os.path.splitext( path )[1][1:] format = path.split('/')[4].split('_')[:2] size = format[0] @@ -3880,10 +3881,8 @@ class YouPornIE(InfoExtractor): return req_format = self._downloader.params.get('format', None) - #format_limit = self._downloader.params.get('format_limit', None) self._downloader.to_screen(u'[youporn] Format: %s' % req_format) - if req_format is None or req_format == 'best': return [formats[0]] elif req_format == 'worst': @@ -3893,74 +3892,43 @@ class YouPornIE(InfoExtractor): else: format = self._specific( req_format, formats ) if result is None: - self._downloader.trouble(u'ERROR: requested format not available') + self._downloader.report_error(u'requested format not available') return return [format] - + class PornotubeIE(InfoExtractor): """Information extractor for pornotube.com.""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$' - IE_NAME = u'pornotube' - VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",' - VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by' - - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_extract_entry(self, url): - """Report downloading extry""" - self._downloader.to_screen(u'[pornotube] Downloading entry: %s' % url.decode('utf-8')) - - def report_date(self, upload_date): - """Report finding uploaded date""" - self._downloader.to_screen(u'[pornotube] Entry date: %s' % upload_date) - - def report_webpage(self, url): - """Report downloading page""" - self._downloader.to_screen(u'[pornotube] Downloaded page: %s' % url) - - def report_title(self, video_title): - """Report downloading extry""" - self._downloader.to_screen(u'[pornotube] Title: %s' % video_title.decode('utf-8')) def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + self._downloader.report_error(u'invalid URL: %s' % url) return - video_id = mobj.group('videoid').decode('utf-8') - video_title = mobj.group('title').decode('utf-8') - self.report_title(video_title); + video_id = mobj.group('videoid') + video_title = mobj.group('title') # Get webpage content - try: - webpage = urllib2.urlopen(url).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err) - return - self.report_webpage(url) + webpage = self._download_webpage(url, video_id) # Get the video URL - result = re.search(self.VIDEO_URL_RE, webpage) + VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",' + result = re.search(VIDEO_URL_RE, webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video url') + self._downloader.report_error(u'unable to extract video url') return - video_url = urllib.unquote(result.group('url').decode('utf-8')) - self.report_extract_entry(video_url) + video_url = compat_urllib_parse.unquote(result.group('url')) #Get the uploaded date - result = re.search(self.VIDEO_UPLOADED_RE, webpage) + VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by' + result = re.search(VIDEO_UPLOADED_RE, webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video title') + self._downloader.report_error(u'unable to extract video title') return - upload_date = result.group('date').decode('utf-8') - self.report_date(upload_date); - + upload_date = result.group('date') info = {'id': video_id, 'url': video_url, @@ -3968,97 +3936,335 @@ class PornotubeIE(InfoExtractor): 'upload_date': upload_date, 'title': video_title, 'ext': 'flv', - 'format': 'flv', - 'thumbnail': None, - 'description': None, - 'player_url': None} + 'format': 'flv'} return [info] - - class YouJizzIE(InfoExtractor): """Information extractor for youjizz.com.""" + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+).html$' - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/([^.]+).html$' - IE_NAME = u'youjizz' - VIDEO_TITLE_RE = r'<title>(?P<title>.*)' - EMBED_PAGE_RE = r'http://www.youjizz.com/videos/embed/(?P[0-9]+)' - SOURCE_RE = r'so.addVariable\("file",encodeURIComponent\("(?P[^"]+)"\)\);' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_extract_entry(self, url): - """Report downloading extry""" - self._downloader.to_screen(u'[youjizz] Downloading entry: %s' % url.decode('utf-8')) - - def report_webpage(self, url): - """Report downloading page""" - self._downloader.to_screen(u'[youjizz] Downloaded page: %s' % url) - - def report_title(self, video_title): - """Report downloading extry""" - self._downloader.to_screen(u'[youjizz] Title: %s' % video_title.decode('utf-8')) + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.report_error(u'invalid URL: %s' % url) + return - def report_embed_page(self, embed_page): - """Report downloading extry""" - self._downloader.to_screen(u'[youjizz] Embed Page: %s' % embed_page.decode('utf-8')) + video_id = mobj.group('videoid') - def _real_extract(self, url): # Get webpage content - try: - webpage = urllib2.urlopen(url).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err) - return - self.report_webpage(url) + webpage = self._download_webpage(url, video_id) # Get the video title - result = re.search(self.VIDEO_TITLE_RE, webpage) + result = re.search(r'(?P<title>.*)', webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = result.group('title').decode('utf-8').strip() - self.report_title(video_title) + raise ExtractorError(u'ERROR: unable to extract video title') + video_title = result.group('title').strip() # Get the embed page - result = re.search(self.EMBED_PAGE_RE, webpage) + result = re.search(r'https?://www.youjizz.com/videos/embed/(?P[0-9]+)', webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract embed page') - return + raise ExtractorError(u'ERROR: unable to extract embed page') + + embed_page_url = result.group(0).strip() + video_id = result.group('videoid') + + webpage = self._download_webpage(embed_page_url, video_id) - embed_page_url = result.group(0).decode('utf-8').strip() - video_id = result.group('videoid').decode('utf-8') - self.report_embed_page(embed_page_url) - - try: - webpage = urllib2.urlopen(embed_page_url).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video embed page: %s' % err) - return - # Get the video URL - result = re.search(self.SOURCE_RE, webpage) + result = re.search(r'so.addVariable\("file",encodeURIComponent\("(?P[^"]+)"\)\);', webpage) if result is None: - self._downloader.trouble(u'ERROR: unable to extract video url') - return - video_url = result.group('source').decode('utf-8') - self.report_extract_entry(video_url) + raise ExtractorError(u'ERROR: unable to extract video url') + video_url = result.group('source') info = {'id': video_id, 'url': video_url, - 'uploader': None, - 'upload_date': None, 'title': video_title, 'ext': 'flv', 'format': 'flv', - 'thumbnail': None, - 'description': None, 'player_url': embed_page_url} return [info] +class EightTracksIE(InfoExtractor): + IE_NAME = '8tracks' + _VALID_URL = r'https?://8tracks.com/(?P[^/]+)/(?P[^/#]+)(?:#.*)?$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + playlist_id = mobj.group('id') + + webpage = self._download_webpage(url, playlist_id) + + m = re.search(r"PAGE.mix = (.*?);\n", webpage, flags=re.DOTALL) + if not m: + raise ExtractorError(u'Cannot find trax information') + json_like = m.group(1) + data = json.loads(json_like) + + session = str(random.randint(0, 1000000000)) + mix_id = data['id'] + track_count = data['tracks_count'] + first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id) + next_url = first_url + res = [] + for i in itertools.count(): + api_json = self._download_webpage(next_url, playlist_id, + note=u'Downloading song information %s/%s' % (str(i+1), track_count), + errnote=u'Failed to download song information') + api_data = json.loads(api_json) + track_data = api_data[u'set']['track'] + info = { + 'id': track_data['id'], + 'url': track_data['track_file_stream_url'], + 'title': track_data['performer'] + u' - ' + track_data['name'], + 'raw_title': track_data['name'], + 'uploader_id': data['user']['login'], + 'ext': 'm4a', + } + res.append(info) + if api_data['set']['at_last_track']: + break + next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id']) + return res + +class KeekIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P\w+)' + IE_NAME = u'keek' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('videoID') + video_url = u'http://cdn.keek.com/keek/video/%s' % video_id + thumbnail = u'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id + webpage = self._download_webpage(url, video_id) + m = re.search(r'[\S\s]+?

(?P.+?)

', webpage) + uploader = clean_html(m.group('uploader')) + info = { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': title, + 'thumbnail': thumbnail, + 'uploader': uploader + } + return [info] + +class TEDIE(InfoExtractor): + _VALID_URL=r'''http://www.ted.com/ + ( + ((?Pplaylists)/(?P\d+)) # We have a playlist + | + ((?Ptalks)) # We have a simple talk + ) + /(?P\w+) # Here goes the name and then ".html" + ''' + + @classmethod + def suitable(cls, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(cls._VALID_URL, url, re.VERBOSE) is not None + + def _real_extract(self, url): + m=re.match(self._VALID_URL, url, re.VERBOSE) + if m.group('type_talk'): + return [self._talk_info(url)] + else : + playlist_id=m.group('playlist_id') + name=m.group('name') + self._downloader.to_screen(u'[%s] Getting info of playlist %s: "%s"' % (self.IE_NAME,playlist_id,name)) + return self._playlist_videos_info(url,name,playlist_id) + + def _talk_video_link(self,mediaSlug): + '''Returns the video link for that mediaSlug''' + return 'http://download.ted.com/talks/%s.mp4' % mediaSlug + + def _playlist_videos_info(self,url,name,playlist_id=0): + '''Returns the videos of the playlist''' + video_RE=r''' + (?P.+?)

' + webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage') + m_videos=re.finditer(video_RE,webpage,re.VERBOSE) + m_names=re.finditer(video_name_RE,webpage) + info=[] + for m_video, m_name in zip(m_videos,m_names): + video_id=m_video.group('video_id') + talk_url='http://www.ted.com%s' % m_name.group('talk_url') + info.append(self._talk_info(talk_url,video_id)) + return info + + def _talk_info(self, url, video_id=0): + """Return the video for the talk in the url""" + m=re.match(self._VALID_URL, url,re.VERBOSE) + videoName=m.group('name') + webpage=self._download_webpage(url, video_id, 'Downloading \"%s\" page' % videoName) + # If the url includes the language we get the title translated + title_RE=r'(?P.*)</span>' + title=re.search(title_RE, webpage).group('title') + info_RE=r'''<script\ type="text/javascript">var\ talkDetails\ =(.*?) + "id":(?P<videoID>[\d]+).*? + "mediaSlug":"(?P<mediaSlug>[\w\d]+?)"''' + thumb_RE=r'</span>[\s.]*</div>[\s.]*<img src="(?P<thumbnail>.*?)"' + thumb_match=re.search(thumb_RE,webpage) + info_match=re.search(info_RE,webpage,re.VERBOSE) + video_id=info_match.group('videoID') + mediaSlug=info_match.group('mediaSlug') + video_url=self._talk_video_link(mediaSlug) + info = { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': title, + 'thumbnail': thumb_match.group('thumbnail') + } + return info + +class MySpassIE(InfoExtractor): + _VALID_URL = r'http://www.myspass.de/.*' + + def _real_extract(self, url): + META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s' + + # video id is the last path element of the URL + # usually there is a trailing slash, so also try the second but last + url_path = compat_urllib_parse_urlparse(url).path + url_parent_path, video_id = os.path.split(url_path) + if not video_id: + _, video_id = os.path.split(url_parent_path) + + # get metadata + metadata_url = META_DATA_URL_TEMPLATE % video_id + metadata_text = self._download_webpage(metadata_url, video_id) + metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8')) + + # extract values from metadata + url_flv_el = metadata.find('url_flv') + if url_flv_el is None: + self._downloader.report_error(u'unable to extract download url') + return + video_url = url_flv_el.text + extension = os.path.splitext(video_url)[1][1:] + title_el = metadata.find('title') + if title_el is None: + self._downloader.report_error(u'unable to extract title') + return + title = title_el.text + format_id_el = metadata.find('format_id') + if format_id_el is None: + format = ext + else: + format = format_id_el.text + description_el = metadata.find('description') + if description_el is not None: + description = description_el.text + else: + description = None + imagePreview_el = metadata.find('imagePreview') + if imagePreview_el is not None: + thumbnail = imagePreview_el.text + else: + thumbnail = None + info = { + 'id': video_id, + 'url': video_url, + 'title': title, + 'ext': extension, + 'format': format, + 'thumbnail': thumbnail, + 'description': description + } + return [info] + +class SpiegelIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('videoID') + + webpage = self._download_webpage(url, video_id) + m = re.search(r'<div class="spVideoTitle">(.*?)</div>', webpage) + if not m: + raise ExtractorError(u'Cannot find title') + video_title = unescapeHTML(m.group(1)) + + xml_url = u'http://video2.spiegel.de/flash/' + video_id + u'.xml' + xml_code = self._download_webpage(xml_url, video_id, + note=u'Downloading XML', errnote=u'Failed to download XML') + + idoc = xml.etree.ElementTree.fromstring(xml_code) + last_type = idoc[-1] + filename = last_type.findall('./filename')[0].text + duration = float(last_type.findall('./duration')[0].text) + + video_url = 'http://video2.spiegel.de/flash/' + filename + video_ext = filename.rpartition('.')[2] + info = { + 'id': video_id, + 'url': video_url, + 'ext': video_ext, + 'title': video_title, + 'duration': duration, + } + return [info] + +class LiveLeakIE(InfoExtractor): + + _VALID_URL = r'^(?:http?://)?(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<video_id>[\w_]+)(?:.*)' + IE_NAME = u'liveleak' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group('video_id') + + webpage = self._download_webpage(url, video_id) + + m = re.search(r'file: "(.*?)",', webpage) + if not m: + self._downloader.report_error(u'unable to find video url') + return + video_url = m.group(1) + + m = re.search(r'<meta property="og:title" content="(?P<title>.*?)"', webpage) + if not m: + self._downloader.trouble(u'Cannot find video title') + title = unescapeHTML(m.group('title')).replace('LiveLeak.com -', '').strip() + + m = re.search(r'<meta property="og:description" content="(?P<desc>.*?)"', webpage) + if m: + desc = unescapeHTML(m.group('desc')) + else: + desc = None + + m = re.search(r'By:.*?(\w+)</a>', webpage) + if m: + uploader = clean_html(m.group(1)) + else: + uploader = None + + info = { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': title, + 'description': desc, + 'uploader': uploader + } + + return [info] + def gen_extractors(): """ Return a list of an instance of every supported extractor. @@ -4099,12 +4305,17 @@ def gen_extractors(): GooglePlusIE(), ArteTvIE(), NBAIE(), + WorldStarHipHopIE(), JustinTVIE(), FunnyOrDieIE(), - TweetReelIE(), SteamIE(), UstreamIE(), + RBMARadioIE(), + EightTracksIE(), + KeekIE(), + TEDIE(), + MySpassIE(), + SpiegelIE(), + LiveLeakIE(), GenericIE() ] - -