X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=61ba2a1b7a61f2c2f3dd760b2db1204e0e460b50;hb=197080b10be36cbeccfe33b8de09a6a8a255f609;hp=9766dea54f94e2b3a38299cf534da6c295d98f3b;hpb=f7b111b7d1b4a7d7d7a8fc99790b3af7d6cff8ad;p=youtube-dl diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 9766dea54..61ba2a1b7 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -3,7 +3,9 @@ from __future__ import absolute_import +import base64 import datetime +import itertools import netrc import os import re @@ -23,7 +25,7 @@ class InfoExtractor(object): Information extractors are the classes that, given a URL, extract information about the video (or videos) the URL refers to. This information includes the real video URL, the video title, author and - others. The information is stored in a dictionary which is then + others. The information is stored in a dictionary which is then passed to the FileDownloader. The FileDownloader processes this information possibly downloading the video to the file system, among other possible outcomes. @@ -32,8 +34,6 @@ class InfoExtractor(object): id: Video identifier. url: Final video URL. - uploader: Nickname of the video uploader, unescaped. - upload_date: Video upload date (YYYYMMDD). title: Video title, unescaped. ext: Video filename extension. @@ -42,6 +42,10 @@ class InfoExtractor(object): format: The video format, defaults to ext (used for --get-format) thumbnail: Full URL to a video thumbnail image. description: One-line video description. + uploader: Full name of the video uploader. + upload_date: Video upload date (YYYYMMDD). + uploader_id: Nickname or id of the video uploader. + location: Physical location of the video. player_url: SWF Player URL (used for rtmpdump). subtitles: The .srt file contents. urlhandle: [internal] The urlHandle to be used to download the file, @@ -100,6 +104,28 @@ class InfoExtractor(object): """Real extraction process. Redefine in subclasses.""" pass + @property + def IE_NAME(self): + return type(self).__name__[:-2] + + def _request_webpage(self, url_or_request, video_id, note=None, errnote=None): + """ Returns the response handle """ + if note is None: + note = u'Downloading video webpage' + self._downloader.to_screen(u'[%s] %s: %s' % (self.IE_NAME, video_id, note)) + try: + return compat_urllib_request.urlopen(url_or_request) + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + if errnote is None: + errnote = u'Unable to download webpage' + raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2]) + + def _download_webpage(self, url_or_request, video_id, note=None, errnote=None): + """ Returns the data of the page as a string """ + urlh = self._request_webpage(url_or_request, video_id, note, errnote) + webpage_bytes = urlh.read() + return webpage_bytes.decode('utf-8', 'replace') + class YoutubeIE(InfoExtractor): """Information extractor for youtube.com.""" @@ -116,7 +142,7 @@ class YoutubeIE(InfoExtractor): |(?: # or the v= param in all its forms (?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx) (?:\?|\#!?) # the params delimiter ? or # or #! - (?:.+&)? # any other preceding param (like /?s=tuff&v=xxxx) + (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx) v= ) )? # optional -> youtube.com/xxxx is OK @@ -125,7 +151,7 @@ class YoutubeIE(InfoExtractor): (?(1).+)? # if we found the ID, everything can follow $""" _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' - _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en' + _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' _NEXT_URL_RE = r'[\?&]next_url=([^&]+)' _NETRC_MACHINE = 'youtube' @@ -159,7 +185,7 @@ class YoutubeIE(InfoExtractor): '44': '480x854', '45': '720x1280', '46': '1080x1920', - } + } IE_NAME = u'youtube' def suitable(self, url): @@ -219,6 +245,39 @@ class YoutubeIE(InfoExtractor): srt += caption + '\n\n' return srt + def _extract_subtitles(self, video_id): + self.report_video_subtitles_download(video_id) + request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) + try: + srt_list = compat_urllib_request.urlopen(request).read().decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None) + srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list) + srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list) + if not srt_lang_list: + return (u'WARNING: video has no closed captions', None) + if self._downloader.params.get('subtitleslang', False): + srt_lang = self._downloader.params.get('subtitleslang') + elif 'en' in srt_lang_list: + srt_lang = 'en' + else: + srt_lang = list(srt_lang_list.keys())[0] + if not srt_lang in srt_lang_list: + return (u'WARNING: no closed captions found in the specified language', None) + params = compat_urllib_parse.urlencode({ + 'lang': srt_lang, + 'name': srt_lang_list[srt_lang].encode('utf-8'), + 'v': video_id, + }) + url = 'http://www.youtube.com/api/timedtext?' + params + try: + srt_xml = compat_urllib_request.urlopen(url).read().decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None) + if not srt_xml: + return (u'WARNING: Did not fetch video subtitles', None) + return (None, self._closed_captions_xml_to_srt(srt_xml)) + def _print_formats(self, formats): print('Available formats:') for x in formats: @@ -261,19 +320,54 @@ class YoutubeIE(InfoExtractor): if username is None: return + request = compat_urllib_request.Request(self._LOGIN_URL) + try: + login_page = compat_urllib_request.urlopen(request).read().decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.to_stderr(u'WARNING: unable to fetch login page: %s' % compat_str(err)) + return + + galx = None + dsh = None + match = re.search(re.compile(r']* name="loginForm"', login_results) is not None: + if re.search(r'(?i)]* id="gaia_loginform"', login_results) is not None: self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') return except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: @@ -293,22 +387,25 @@ class YoutubeIE(InfoExtractor): self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) return - def _real_extract(self, url): - # Extract original video URL from URL with redirection, like age verification, using next_url parameter - mobj = re.search(self._NEXT_URL_RE, url) - if mobj: - url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/') - - # Extract video id from URL + def _extract_id(self, url): mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return video_id = mobj.group(2) + return video_id + + def _real_extract(self, url): + # Extract original video URL from URL with redirection, like age verification, using next_url parameter + mobj = re.search(self._NEXT_URL_RE, url) + if mobj: + url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/') + video_id = self._extract_id(url) # Get video webpage self.report_video_webpage_download(video_id) - request = compat_urllib_request.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id) + url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id + request = compat_urllib_request.Request(url) try: video_webpage_bytes = compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: @@ -356,10 +453,18 @@ class YoutubeIE(InfoExtractor): # uploader if 'author' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') + self._downloader.trouble(u'ERROR: unable to extract uploader name') return video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0]) + # uploader_id + video_uploader_id = None + mobj = re.search(r'', video_webpage) + if mobj is not None: + video_uploader_id = mobj.group(1) + else: + self._downloader.trouble(u'WARNING: unable to extract uploader nickname') + # title if 'title' not in video_info: self._downloader.trouble(u'ERROR: unable to extract video title') @@ -395,35 +500,9 @@ class YoutubeIE(InfoExtractor): # closed captions video_subtitles = None if self._downloader.params.get('writesubtitles', False): - try: - self.report_video_subtitles_download(video_id) - request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) - try: - srt_list = compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) - srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list) - srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list) - if not srt_lang_list: - raise Trouble(u'WARNING: video has no closed captions') - if self._downloader.params.get('subtitleslang', False): - srt_lang = self._downloader.params.get('subtitleslang') - elif 'en' in srt_lang_list: - srt_lang = 'en' - else: - srt_lang = srt_lang_list.keys()[0] - if not srt_lang in srt_lang_list: - raise Trouble(u'WARNING: no closed captions found in the specified language') - request = compat_urllib_request.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id)) - try: - srt_xml = compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) - if not srt_xml: - raise Trouble(u'WARNING: unable to download video subtitles') - video_subtitles = self._closed_captions_xml_to_srt(srt_xml) - except Trouble as trouble: - self._downloader.trouble(str(trouble)) + (srt_error, video_subtitles) = self._extract_subtitles(video_id) + if srt_error: + self._downloader.trouble(srt_error) if 'length_seconds' not in video_info: self._downloader.trouble(u'WARNING: unable to extract video duration') @@ -443,7 +522,7 @@ class YoutubeIE(InfoExtractor): elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1: url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',') url_data = [compat_parse_qs(uds) for uds in url_data_strs] - url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data) + url_data = [ud for ud in url_data if 'itag' in ud and 'url' in ud] url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data) format_limit = self._downloader.params.get('format_limit', None) @@ -493,6 +572,7 @@ class YoutubeIE(InfoExtractor): 'id': video_id, 'url': video_real_url, 'uploader': video_uploader, + 'uploader_id': video_uploader_id, 'upload_date': upload_date, 'title': video_title, 'ext': video_extension, @@ -638,14 +718,11 @@ class DailymotionIE(InfoExtractor): _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)' IE_NAME = u'dailymotion' + _WORKING = False def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id) - def report_extraction(self, video_id): """Report information extraction.""" self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id) @@ -664,13 +741,7 @@ class DailymotionIE(InfoExtractor): # Retrieve video webpage to extract further information request = compat_urllib_request.Request(url) request.add_header('Cookie', 'family_filter=off') - try: - self.report_download_webpage(video_id) - webpage_bytes = compat_urllib_request.urlopen(request).read() - webpage = webpage_bytes.decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) - return + webpage = self._download_webpage(request, video_id) # Extract URL, uploader and title from webpage self.report_extraction(video_id) @@ -799,6 +870,7 @@ class PhotobucketIE(InfoExtractor): class YahooIE(InfoExtractor): """Information extractor for video.yahoo.com.""" + _WORKING = False # _VALID_URL matches all Yahoo! Video URLs # _VPAGE_URL matches only the extractable '/watch/' URLs _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?' @@ -942,7 +1014,7 @@ class VimeoIE(InfoExtractor): """Information extractor for vimeo.com.""" # _VALID_URL matches Vimeo URLs - _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?:videos?/)?([0-9]+)' + _VALID_URL = r'(?Phttps?://)?(?:(?:www|player)\.)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)' IE_NAME = u'vimeo' def __init__(self, downloader=None): @@ -963,7 +1035,11 @@ class VimeoIE(InfoExtractor): self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) return - video_id = mobj.group(1) + video_id = mobj.group('id') + if not mobj.group('proto'): + url = 'https://' + url + if mobj.group('direct_link'): + url = 'https://vimeo.com/' + video_id # Retrieve video webpage to extract further information request = compat_urllib_request.Request(url, None, std_headers) @@ -987,26 +1063,27 @@ class VimeoIE(InfoExtractor): except: self._downloader.trouble(u'ERROR: unable to extract info section') return - + # Extract title video_title = config["video"]["title"] - # Extract uploader + # Extract uploader and uploader_id video_uploader = config["video"]["owner"]["name"] + video_uploader_id = config["video"]["owner"]["url"].split('/')[-1] # Extract video thumbnail video_thumbnail = config["video"]["thumbnail"] # Extract video description - video_description = get_element_by_id("description", webpage) + video_description = get_element_by_attribute("itemprop", "description", webpage) if video_description: video_description = clean_html(video_description) else: video_description = '' # Extract upload date video_upload_date = None - mobj = re.search(r'', webpage) + mobj = re.search(r'\d+)(?:.*)' _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&' _NETRC_MACHINE = 'facebook' - _available_formats = ['video', 'highqual', 'lowqual'] - _video_extensions = { - 'video': 'mp4', - 'highqual': 'mp4', - 'lowqual': 'mp4', - } IE_NAME = u'facebook' - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def _reporter(self, message): - """Add header and report message.""" - self._downloader.to_screen(u'[facebook] %s' % message) - def report_login(self): """Report attempt to log in.""" - self._reporter(u'Logging in') - - def report_video_webpage_download(self, video_id): - """Report attempt to download video webpage.""" - self._reporter(u'%s: Downloading video webpage' % video_id) - - def report_information_extraction(self, video_id): - """Report attempt to extract video information.""" - self._reporter(u'%s: Extracting video information' % video_id) - - def _parse_page(self, video_webpage): - """Extract video information from page""" - # General data - data = {'title': r'\("video_title", "(.*?)"\)', - 'description': r'
(.*?)
', - 'owner': r'\("video_owner_name", "(.*?)"\)', - 'thumbnail': r'\("thumb_url", "(?P.*?)"\)', - } - video_info = {} - for piece in data.keys(): - mobj = re.search(data[piece], video_webpage) - if mobj is not None: - video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) - - # Video urls - video_urls = {} - for fmt in self._available_formats: - mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage) - if mobj is not None: - # URL is in a Javascript segment inside an escaped Unicode format within - # the generally utf-8 page - video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) - video_info['video_urls'] = video_urls - - return video_info + self._downloader.to_screen(u'[%s] Logging in' % self.IE_NAME) def _real_initialize(self): if self._downloader is None: @@ -2059,100 +2086,39 @@ class FacebookIE(InfoExtractor): return video_id = mobj.group('ID') - # Get video webpage - self.report_video_webpage_download(video_id) - request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id) - try: - page = compat_urllib_request.urlopen(request) - video_webpage = page.read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - return - - # Start extracting information - self.report_information_extraction(video_id) - - # Extract information - video_info = self._parse_page(video_webpage) - - # uploader - if 'owner' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') - return - video_uploader = video_info['owner'] - - # title - if 'title' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = video_info['title'] - video_title = video_title.decode('utf-8') - - # thumbnail image - if 'thumbnail' not in video_info: - self._downloader.trouble(u'WARNING: unable to extract video thumbnail') - video_thumbnail = '' - else: - video_thumbnail = video_info['thumbnail'] + url = 'https://www.facebook.com/video/video.php?v=%s' % video_id + webpage = self._download_webpage(url, video_id) + + BEFORE = '[["allowFullScreen","true"],["allowScriptAccess","always"],["salign","tl"],["scale","noscale"],["wmode","opaque"]].forEach(function(param) {swf.addParam(param[0], param[1]);});\n' + AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});' + m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage) + if not m: + raise ExtractorError(u'Cannot parse data') + data = dict(json.loads(m.group(1))) + params_raw = compat_urllib_parse.unquote(data['params']) + params = json.loads(params_raw) + video_url = params['hd_src'] + if not video_url: + video_url = params['sd_src'] + if not video_url: + raise ExtractorError(u'Cannot find video URL') + video_duration = int(params['video_duration']) + + m = re.search('

([^<]+)

', webpage) + if not m: + raise ExtractorError(u'Cannot find title in webpage') + video_title = unescapeHTML(m.group(1)) - # upload date - upload_date = None - if 'upload_date' in video_info: - upload_time = video_info['upload_date'] - timetuple = email.utils.parsedate_tz(upload_time) - if timetuple is not None: - try: - upload_date = time.strftime('%Y%m%d', timetuple[0:9]) - except: - pass - - # description - video_description = video_info.get('description', 'No description available.') - - url_map = video_info['video_urls'] - if len(url_map.keys()) > 0: - # Decide which formats to download - req_format = self._downloader.params.get('format', None) - format_limit = self._downloader.params.get('format_limit', None) - - if format_limit is not None and format_limit in self._available_formats: - format_list = self._available_formats[self._available_formats.index(format_limit):] - else: - format_list = self._available_formats - existing_formats = [x for x in format_list if x in url_map] - if len(existing_formats) == 0: - self._downloader.trouble(u'ERROR: no known formats available for video') - return - if req_format is None: - video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality - elif req_format == 'worst': - video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality - elif req_format == '-1': - video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats - else: - # Specific format - if req_format not in url_map: - self._downloader.trouble(u'ERROR: requested format not available') - return - video_url_list = [(req_format, url_map[req_format])] # Specific format + info = { + 'id': video_id, + 'title': video_title, + 'url': video_url, + 'ext': 'mp4', + 'duration': video_duration, + 'thumbnail': params['thumbnail_src'], + } + return [info] - results = [] - for format_param, video_real_url in video_url_list: - # Extension - video_extension = self._video_extensions.get(format_param, 'mp4') - - results.append({ - 'id': video_id.decode('utf-8'), - 'url': video_real_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), - 'upload_date': upload_date, - 'title': video_title, - 'ext': video_extension.decode('utf-8'), - 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), - 'thumbnail': video_thumbnail.decode('utf-8'), - 'description': video_description.decode('utf-8'), - }) - return results class BlipTVIE(InfoExtractor): """Information extractor for blip.tv""" @@ -2181,6 +2147,7 @@ class BlipTVIE(InfoExtractor): cchar = '?' json_url = url + cchar + 'skin=json&version=2&no_wrap=1' request = compat_urllib_request.Request(json_url) + request.add_header('User-Agent', 'iTunes/10.6.1') self.report_extraction(mobj.group(1)) info = None try: @@ -2201,8 +2168,7 @@ class BlipTVIE(InfoExtractor): 'urlhandle': urlh } except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) - return + raise ExtractorError(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) if info is None: # Regular URL try: json_code_bytes = urlh.read() @@ -2235,13 +2201,13 @@ class BlipTVIE(InfoExtractor): 'format': data['media']['mimeType'], 'thumbnail': data['thumbnailUrl'], 'description': data['description'], - 'player_url': data['embedUrl'] + 'player_url': data['embedUrl'], + 'user_agent': 'iTunes/10.6.1', } except (ValueError,KeyError) as err: self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err)) return - std_headers['User-Agent'] = 'iTunes/10.6.1' return [info] @@ -2253,10 +2219,6 @@ class MyVideoIE(InfoExtractor): def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id) def report_extraction(self, video_id): """Report information extraction.""" @@ -2271,16 +2233,11 @@ class MyVideoIE(InfoExtractor): video_id = mobj.group(1) # Get video webpage - request = compat_urllib_request.Request('http://www.myvideo.de/watch/%s' % video_id) - try: - self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) - return + webpage_url = 'http://www.myvideo.de/watch/%s' % video_id + webpage = self._download_webpage(webpage_url, video_id) self.report_extraction(video_id) - mobj = re.search(r'', + mobj = re.search(r'', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract media URL') @@ -2307,10 +2264,10 @@ class ComedyCentralIE(InfoExtractor): """Information extractor for The Daily Show and Colbert Report """ # urls can be abbreviations like :thedailyshow or :colbert - # urls for episodes like: + # urls for episodes like: # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news - # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524 + # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524 _VALID_URL = r"""^(:(?Ptds|thedailyshow|cr|colbert|colbertnation|colbertreport) |(https?://)?(www\.)? (?Pthedailyshow|colbertnation)\.com/ @@ -2318,8 +2275,7 @@ class ComedyCentralIE(InfoExtractor): (?P (the-colbert-report-(videos|collections)/(?P[0-9]+)/[^/]*/(?P.*?)) |(watch/(?P[^/]*)/(?P.*))))) - $""" - IE_NAME = u'comedycentral' + $""" _available_formats = ['3500', '2200', '1700', '1200', '750', '400'] @@ -2347,16 +2303,12 @@ class ComedyCentralIE(InfoExtractor): def report_extraction(self, episode_id): self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) - def report_config_download(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id) + def report_config_download(self, episode_id, media_id): + self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration for %s' % (episode_id, media_id)) def report_index_download(self, episode_id): self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id) - def report_player_url(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id) - - def _print_formats(self, formats): print('Available formats:') for x in formats: @@ -2395,6 +2347,7 @@ class ComedyCentralIE(InfoExtractor): try: htmlHandle = compat_urllib_request.urlopen(req) html = htmlHandle.read() + webpage = html.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) return @@ -2409,28 +2362,19 @@ class ComedyCentralIE(InfoExtractor): return epTitle = mobj.group('episode') - mMovieParams = re.findall('(?:gsp.comedystor/.*)$', rtmp_video_url) + if not m: + raise ExtractorError(u'Cannot transform RTMP url') + base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/' + video_url = base + m.group('finalid') - if video_url.startswith(broken_cdn): - video_url = video_url.replace(broken_cdn, better_cdn) - - effTitle = showId + u'-' + epTitle + effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1) info = { 'id': shortMediaId, 'url': video_url, @@ -2507,11 +2449,9 @@ class ComedyCentralIE(InfoExtractor): 'format': format, 'thumbnail': None, 'description': officialTitle, - 'player_url': None #playerUrl } - results.append(info) - + return results @@ -2556,7 +2496,9 @@ class EscapistIE(InfoExtractor): self.report_config_download(showName) try: - configJSON = compat_urllib_request.urlopen(configUrl).read() + configJSON = compat_urllib_request.urlopen(configUrl) + m = re.match(r'text/html; charset="?([^"]+)"?', configJSON.headers['Content-Type']) + configJSON = configJSON.read().decode(m.group(1) if m else 'utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err)) return @@ -2587,7 +2529,6 @@ class EscapistIE(InfoExtractor): return [info] - class CollegeHumorIE(InfoExtractor): """Information extractor for collegehumor.com""" @@ -2666,10 +2607,6 @@ class XVideosIE(InfoExtractor): _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)' IE_NAME = u'xvideos' - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) - def report_extraction(self, video_id): """Report information extraction.""" self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) @@ -2681,15 +2618,7 @@ class XVideosIE(InfoExtractor): return video_id = mobj.group(1) - self.report_webpage(video_id) - - request = compat_urllib_request.Request(r'http://www.xvideos.com/video' + video_id) - try: - webpage_bytes = compat_urllib_request.urlopen(request).read() - webpage = webpage_bytes.decode('utf-8', 'replace') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - return + webpage = self._download_webpage(url, video_id) self.report_extraction(video_id) @@ -2788,7 +2717,7 @@ class SoundcloudIE(InfoExtractor): stream_json_bytes = compat_urllib_request.urlopen(request).read() stream_json = stream_json_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) + self._downloader.trouble(u'ERROR: unable to download stream definitions: %s' % compat_str(err)) return streams = json.loads(stream_json) @@ -2807,13 +2736,7 @@ class SoundcloudIE(InfoExtractor): class InfoQIE(InfoExtractor): """Information extractor for infoq.com""" - _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$' - IE_NAME = u'infoq' - - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) def report_extraction(self, video_id): """Report information extraction.""" @@ -2825,38 +2748,29 @@ class InfoQIE(InfoExtractor): self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return - self.report_webpage(url) - - request = compat_urllib_request.Request(url) - try: - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - return - + webpage = self._download_webpage(url, video_id=url) self.report_extraction(url) - # Extract video URL mobj = re.search(r"jsclassref='([^']*)'", webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video url') return - video_url = 'rtmpe://video.infoq.com/cfx/st/' + compat_urllib_parse.unquote(mobj.group(1).decode('base64')) - + real_id = compat_urllib_parse.unquote(base64.b64decode(mobj.group(1).encode('ascii')).decode('utf-8')) + video_url = 'rtmpe://video.infoq.com/cfx/st/' + real_id # Extract title mobj = re.search(r'contentTitle = "(.*?)";', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video title') return - video_title = mobj.group(1).decode('utf-8') + video_title = mobj.group(1) # Extract description video_description = u'No description available.' mobj = re.search(r'', webpage) if mobj is not None: - video_description = mobj.group(1).decode('utf-8') + video_description = mobj.group(1) video_filename = video_url.split('/')[-1] video_id, extension = video_filename.split('.') @@ -2876,6 +2790,8 @@ class InfoQIE(InfoExtractor): class MixcloudIE(InfoExtractor): """Information extractor for www.mixcloud.com""" + + _WORKING = False # New API, but it seems good http://www.mixcloud.com/developers/documentation/ _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)' IE_NAME = u'mixcloud' @@ -2966,7 +2882,7 @@ class MixcloudIE(InfoExtractor): if file_url is not None: break # got it! else: - if req_format not in formats.keys(): + if req_format not in formats: self._downloader.trouble(u'ERROR: format is not available') return @@ -3004,8 +2920,7 @@ class StanfordOpenClassroomIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return + raise ExtractorError(u'Invalid URL: %s' % url) if mobj.group('course') and mobj.group('video'): # A specific video course = mobj.group('course') @@ -3042,12 +2957,9 @@ class StanfordOpenClassroomIE(InfoExtractor): 'upload_date': None, } - self.report_download_webpage(info['id']) - try: - coursepage = compat_urllib_request.urlopen(url).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err)) - return + coursepage = self._download_webpage(url, info['id'], + note='Downloading course info page', + errnote='Unable to download course info page') m = re.search('

([^<]+)

', coursepage) if m: @@ -3071,7 +2983,6 @@ class StanfordOpenClassroomIE(InfoExtractor): assert entry['type'] == 'reference' results += self.extract(entry['url']) return results - else: # Root page info = { 'id': 'Stanford OpenClassroom', @@ -3110,10 +3021,6 @@ class MTVIE(InfoExtractor): _VALID_URL = r'^(?Phttps?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P[0-9]+)/[^/]+$' IE_NAME = u'mtv' - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) - def report_extraction(self, video_id): """Report information extraction.""" self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) @@ -3126,14 +3033,8 @@ class MTVIE(InfoExtractor): if not mobj.group('proto'): url = 'http://' + url video_id = mobj.group('videoid') - self.report_webpage(video_id) - request = compat_urllib_request.Request(url) - try: - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - return + webpage = self._download_webpage(url, video_id) mobj = re.search(r'', webpage) if mobj is None: @@ -3145,7 +3046,7 @@ class MTVIE(InfoExtractor): self._downloader.trouble(u'ERROR: unable to extract performer') return performer = unescapeHTML(mobj.group(1).decode('iso-8859-1')) - video_title = performer + ' - ' + song_name + video_title = performer + ' - ' + song_name mobj = re.search(r'', webpage) if mobj is None: @@ -3196,20 +3097,15 @@ class MTVIE(InfoExtractor): class YoukuIE(InfoExtractor): - _VALID_URL = r'(?:http://)?v\.youku\.com/v_show/id_(?P[A-Za-z0-9]+)\.html' - IE_NAME = u'Youku' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) def report_download_webpage(self, file_id): """Report webpage download.""" - self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id) + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, file_id)) def report_extraction(self, file_id): """Report information extraction.""" - self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id) + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id)) def _gen_sid(self): nowTime = int(time.time() * 1000) @@ -3265,7 +3161,7 @@ class YoukuIE(InfoExtractor): seed = config['data'][0]['seed'] format = self._downloader.params.get('format', None) - supported_format = config['data'][0]['streamfileids'].keys() + supported_format = list(config['data'][0]['streamfileids'].keys()) if format is None or format == 'best': if 'hd2' in supported_format: @@ -3314,7 +3210,7 @@ class YoukuIE(InfoExtractor): class XNXXIE(InfoExtractor): """Information extractor for xnxx.com""" - _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)' + _VALID_URL = r'^(?:https?://)?video\.xnxx\.com/video([0-9]+)/(.*)' IE_NAME = u'xnxx' VIDEO_URL_RE = r'flv_url=(.*?)&' VIDEO_TITLE_RE = r'(.*?)\s+-\s+XNXX.COM' @@ -3378,7 +3274,7 @@ class XNXXIE(InfoExtractor): class GooglePlusIE(InfoExtractor): """Information extractor for plus.google.com.""" - _VALID_URL = r'(?:https://)?plus\.google\.com/(?:\w+/)*?(\d+)/posts/(\w+)' + _VALID_URL = r'(?:https://)?plus\.google\.com/(?:[^/]+/)*?posts/(\w+)' IE_NAME = u'plus.google' def __init__(self, downloader=None): @@ -3386,7 +3282,7 @@ class GooglePlusIE(InfoExtractor): def report_extract_entry(self, url): """Report downloading extry""" - self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url.decode('utf-8')) + self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url) def report_date(self, upload_date): """Report downloading extry""" @@ -3394,15 +3290,15 @@ class GooglePlusIE(InfoExtractor): def report_uploader(self, uploader): """Report downloading extry""" - self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader.decode('utf-8')) + self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader) def report_title(self, video_title): """Report downloading extry""" - self._downloader.to_screen(u'[plus.google] Title: %s' % video_title.decode('utf-8')) + self._downloader.to_screen(u'[plus.google] Title: %s' % video_title) def report_extract_vid_page(self, video_page): """Report information extraction.""" - self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page.decode('utf-8')) + self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page) def _real_extract(self, url): # Extract id from URL @@ -3412,7 +3308,7 @@ class GooglePlusIE(InfoExtractor): return post_url = mobj.group(0) - video_id = mobj.group(2) + video_id = mobj.group(1) video_extension = 'flv' @@ -3420,7 +3316,7 @@ class GooglePlusIE(InfoExtractor): self.report_extract_entry(post_url) request = compat_urllib_request.Request(post_url) try: - webpage = compat_urllib_request.urlopen(request).read() + webpage = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err)) return @@ -3462,7 +3358,7 @@ class GooglePlusIE(InfoExtractor): video_page = mobj.group(1) request = compat_urllib_request.Request(video_page) try: - webpage = compat_urllib_request.urlopen(request).read() + webpage = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) return @@ -3484,25 +3380,25 @@ class GooglePlusIE(InfoExtractor): # Only get the url. The resolution part in the tuple has no use anymore video_url = video_url[-1] # Treat escaped \u0026 style hex - video_url = unicode(video_url, "unicode_escape") + try: + video_url = video_url.decode("unicode_escape") + except AttributeError: # Python 3 + video_url = bytes(video_url, 'ascii').decode('unicode-escape') return [{ - 'id': video_id.decode('utf-8'), + 'id': video_id, 'url': video_url, - 'uploader': uploader.decode('utf-8'), - 'upload_date': upload_date.decode('utf-8'), - 'title': video_title.decode('utf-8'), - 'ext': video_extension.decode('utf-8'), + 'uploader': uploader, + 'upload_date': upload_date, + 'title': video_title, + 'ext': video_extension, }] class NBAIE(InfoExtractor): _VALID_URL = r'^(?:https?://)?(?:watch\.|www\.)?nba\.com/(?:nba/)?video(/[^?]*)(\?.*)?$' IE_NAME = u'nba' - def report_extraction(self, video_id): - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) - def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: @@ -3513,14 +3409,7 @@ class NBAIE(InfoExtractor): if video_id.endswith('/index.html'): video_id = video_id[:-len('/index.html')] - self.report_extraction(video_id) - try: - urlh = compat_urllib_request.urlopen(url) - webpage_bytes = urlh.read() - webpage = webpage_bytes.decode('utf-8', 'ignore') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) - return + webpage = self._download_webpage(url, video_id) video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4' def _findProp(rexp, default=None): @@ -3571,19 +3460,27 @@ class JustinTVIE(InfoExtractor): except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video info JSON: %s' % compat_str(err)) return - + response = json.loads(webpage) + if type(response) != list: + error_text = response.get('error', 'unknown error') + self._downloader.trouble(u'ERROR: Justin.tv API: %s' % error_text) + return info = [] for clip in response: video_url = clip['video_file_url'] if video_url: video_extension = os.path.splitext(video_url)[1][1:] - video_date = re.sub('-', '', clip['created_on'][:10]) + video_date = re.sub('-', '', clip['start_time'][:10]) + video_uploader_id = clip.get('user_id', clip.get('channel_id')) + video_id = clip['id'] + video_title = clip.get('title', video_id) info.append({ - 'id': clip['id'], + 'id': video_id, 'url': video_url, - 'title': clip['title'], - 'uploader': clip.get('user_id', clip.get('channel_id')), + 'title': video_title, + 'uploader': clip.get('channel_name', video_uploader_id), + 'uploader_id': video_uploader_id, 'upload_date': video_date, 'ext': video_extension, }) @@ -3594,7 +3491,7 @@ class JustinTVIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return - + api = 'http://api.justin.tv' video_id = mobj.group(mobj.lastindex) paged = False @@ -3602,11 +3499,11 @@ class JustinTVIE(InfoExtractor): paged = True api += '/channel/archives/%s.json' else: - api += '/clip/show/%s.json' + api += '/broadcast/by_archive/%s.json' api = api % (video_id,) - + self.report_extraction(video_id) - + info = [] offset = 0 limit = self._JUSTIN_PAGE_LIMIT @@ -3620,3 +3517,640 @@ class JustinTVIE(InfoExtractor): break offset += limit return info + +class FunnyOrDieIE(InfoExtractor): + _VALID_URL = r'^(?:https?://)?(?:www\.)?funnyordie\.com/videos/(?P<id>[0-9a-f]+)/.*$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + + m = re.search(r'<video[^>]*>\s*<source[^>]*>\s*<source src="(?P<url>[^"]+)"', webpage, re.DOTALL) + if not m: + self._downloader.trouble(u'ERROR: unable to find video information') + video_url = unescapeHTML(m.group('url')) + + m = re.search(r"class='player_page_h1'>\s+<a.*?>(?P<title>.*?)</a>", webpage) + if not m: + self._downloader.trouble(u'Cannot find video title') + title = unescapeHTML(m.group('title')) + + m = re.search(r'<meta property="og:description" content="(?P<desc>.*?)"', webpage) + if m: + desc = unescapeHTML(m.group('desc')) + else: + desc = None + + info = { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': title, + 'description': desc, + } + return [info] + +class TweetReelIE(InfoExtractor): + _VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P<id>[0-9a-z]+)$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + + m = re.search(r'<div id="left" status_id="([0-9]+)">', webpage) + if not m: + self._downloader.trouble(u'ERROR: Cannot find status ID') + status_id = m.group(1) + + m = re.search(r'<div class="tweet_text">(.*?)</div>', webpage, flags=re.DOTALL) + if not m: + self._downloader.trouble(u'WARNING: Cannot find description') + desc = unescapeHTML(re.sub('<a.*?</a>', '', m.group(1))).strip() + + m = re.search(r'<div class="tweet_info">.*?from <a target="_blank" href="https?://twitter.com/(?P<uploader_id>.+?)">(?P<uploader>.+?)</a>', webpage, flags=re.DOTALL) + if not m: + self._downloader.trouble(u'ERROR: Cannot find uploader') + uploader = unescapeHTML(m.group('uploader')) + uploader_id = unescapeHTML(m.group('uploader_id')) + + m = re.search(r'<span unixtime="([0-9]+)"', webpage) + if not m: + self._downloader.trouble(u'ERROR: Cannot find upload date') + upload_date = datetime.datetime.fromtimestamp(int(m.group(1))).strftime('%Y%m%d') + + title = desc + video_url = 'http://files.tweetreel.com/video/' + status_id + '.mov' + + info = { + 'id': video_id, + 'url': video_url, + 'ext': 'mov', + 'title': title, + 'description': desc, + 'uploader': uploader, + 'uploader_id': uploader_id, + 'internal_id': status_id, + 'upload_date': upload_date + } + return [info] + +class SteamIE(InfoExtractor): + _VALID_URL = r"""http://store.steampowered.com/ + (?P<urltype>video|app)/ #If the page is only for videos or for a game + (?P<gameID>\d+)/? + (?P<videoID>\d*)(?P<extra>\??) #For urltype == video we sometimes get the videoID + """ + + def suitable(self, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(self._VALID_URL, url, re.VERBOSE) is not None + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url, re.VERBOSE) + urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\}," + gameID = m.group('gameID') + videourl = 'http://store.steampowered.com/video/%s/' % gameID + webpage = self._download_webpage(videourl, gameID) + mweb = re.finditer(urlRE, webpage) + namesRE = r'<span class="title">(?P<videoName>.+?)</span>' + titles = re.finditer(namesRE, webpage) + videos = [] + for vid,vtitle in zip(mweb,titles): + video_id = vid.group('videoID') + title = vtitle.group('videoName') + video_url = vid.group('videoURL') + if not video_url: + self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id) + info = { + 'id':video_id, + 'url':video_url, + 'ext': 'flv', + 'title': unescapeHTML(title) + } + videos.append(info) + return videos + +class UstreamIE(InfoExtractor): + _VALID_URL = r'https?://www\.ustream\.tv/recorded/(?P<videoID>\d+)' + IE_NAME = u'ustream' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('videoID') + video_url = u'http://tcdn.ustream.tv/video/%s' % video_id + webpage = self._download_webpage(url, video_id) + m = re.search(r'data-title="(?P<title>.+)"',webpage) + title = m.group('title') + m = re.search(r'<a class="state" data-content-type="channel" data-content-id="(?P<uploader>\d+)"',webpage) + uploader = m.group('uploader') + info = { + 'id':video_id, + 'url':video_url, + 'ext': 'flv', + 'title': title, + 'uploader': uploader + } + return [info] + +class RBMARadioIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<videoID>[^/]+)$' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('videoID') + + webpage = self._download_webpage(url, video_id) + m = re.search(r'<script>window.gon = {.*?};gon\.show=(.+?);</script>', webpage) + if not m: + raise ExtractorError(u'Cannot find metadata') + json_data = m.group(1) + + try: + data = json.loads(json_data) + except ValueError as e: + raise ExtractorError(u'Invalid JSON: ' + str(e)) + + video_url = data['akamai_url'] + '&cbr=256' + url_parts = compat_urllib_parse_urlparse(video_url) + video_ext = url_parts.path.rpartition('.')[2] + info = { + 'id': video_id, + 'url': video_url, + 'ext': video_ext, + 'title': data['title'], + 'description': data.get('teaser_text'), + 'location': data.get('country_of_origin'), + 'uploader': data.get('host', {}).get('name'), + 'uploader_id': data.get('host', {}).get('slug'), + 'thumbnail': data.get('image', {}).get('large_url_2x'), + 'duration': data.get('duration'), + } + return [info] + + +class YouPornIE(InfoExtractor): + """Information extractor for youporn.com.""" + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+)' + + def _print_formats(self, formats): + """Print all available formats""" + print(u'Available formats:') + print(u'ext\t\tformat') + print(u'---------------------------------') + for format in formats: + print(u'%s\t\t%s' % (format['ext'], format['format'])) + + def _specific(self, req_format, formats): + for x in formats: + if(x["format"]==req_format): + return x + return None + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group('videoid') + + req = compat_urllib_request.Request(url) + req.add_header('Cookie', 'age_verified=1') + webpage = self._download_webpage(req, video_id) + + # Get the video title + result = re.search(r'videoTitleArea">(?P<title>.*)</h1>', webpage) + if result is None: + raise ExtractorError(u'ERROR: unable to extract video title') + video_title = result.group('title').strip() + + # Get the video date + result = re.search(r'Date:</b>(?P<date>.*)</li>', webpage) + if result is None: + self._downloader.to_stderr(u'WARNING: unable to extract video date') + upload_date = None + else: + upload_date = result.group('date').strip() + + # Get the video uploader + result = re.search(r'Submitted:</b>(?P<uploader>.*)</li>', webpage) + if result is None: + self._downloader.to_stderr(u'ERROR: unable to extract uploader') + video_uploader = None + else: + video_uploader = result.group('uploader').strip() + video_uploader = clean_html( video_uploader ) + + # Get all of the formats available + DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>' + result = re.search(DOWNLOAD_LIST_RE, webpage) + if result is None: + raise ExtractorError(u'Unable to extract download list') + download_list_html = result.group('download_list').strip() + + # Get all of the links from the page + LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">' + links = re.findall(LINK_RE, download_list_html) + if(len(links) == 0): + raise ExtractorError(u'ERROR: no known formats available for video') + + self._downloader.to_screen(u'[youporn] Links found: %d' % len(links)) + + formats = [] + for link in links: + + # A link looks like this: + # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0 + # A path looks like this: + # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4 + video_url = unescapeHTML( link ) + path = compat_urllib_parse_urlparse( video_url ).path + extension = os.path.splitext( path )[1][1:] + format = path.split('/')[4].split('_')[:2] + size = format[0] + bitrate = format[1] + format = "-".join( format ) + title = u'%s-%s-%s' % (video_title, size, bitrate) + + formats.append({ + 'id': video_id, + 'url': video_url, + 'uploader': video_uploader, + 'upload_date': upload_date, + 'title': title, + 'ext': extension, + 'format': format, + 'thumbnail': None, + 'description': None, + 'player_url': None + }) + + if self._downloader.params.get('listformats', None): + self._print_formats(formats) + return + + req_format = self._downloader.params.get('format', None) + self._downloader.to_screen(u'[youporn] Format: %s' % req_format) + + if req_format is None or req_format == 'best': + return [formats[0]] + elif req_format == 'worst': + return [formats[-1]] + elif req_format in ('-1', 'all'): + return formats + else: + format = self._specific( req_format, formats ) + if result is None: + self._downloader.trouble(u'ERROR: requested format not available') + return + return [format] + + + +class PornotubeIE(InfoExtractor): + """Information extractor for pornotube.com.""" + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group('videoid') + video_title = mobj.group('title') + + # Get webpage content + webpage = self._download_webpage(url, video_id) + + # Get the video URL + VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",' + result = re.search(VIDEO_URL_RE, webpage) + if result is None: + self._downloader.trouble(u'ERROR: unable to extract video url') + return + video_url = compat_urllib_parse.unquote(result.group('url')) + + #Get the uploaded date + VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by' + result = re.search(VIDEO_UPLOADED_RE, webpage) + if result is None: + self._downloader.trouble(u'ERROR: unable to extract video title') + return + upload_date = result.group('date') + + info = {'id': video_id, + 'url': video_url, + 'uploader': None, + 'upload_date': upload_date, + 'title': video_title, + 'ext': 'flv', + 'format': 'flv'} + + return [info] + +class YouJizzIE(InfoExtractor): + """Information extractor for youjizz.com.""" + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+).html$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group('videoid') + + # Get webpage content + webpage = self._download_webpage(url, video_id) + + # Get the video title + result = re.search(r'<title>(?P<title>.*)', webpage) + if result is None: + raise ExtractorError(u'ERROR: unable to extract video title') + video_title = result.group('title').strip() + + # Get the embed page + result = re.search(r'https?://www.youjizz.com/videos/embed/(?P[0-9]+)', webpage) + if result is None: + raise ExtractorError(u'ERROR: unable to extract embed page') + + embed_page_url = result.group(0).strip() + video_id = result.group('videoid') + + webpage = self._download_webpage(embed_page_url, video_id) + + # Get the video URL + result = re.search(r'so.addVariable\("file",encodeURIComponent\("(?P[^"]+)"\)\);', webpage) + if result is None: + raise ExtractorError(u'ERROR: unable to extract video url') + video_url = result.group('source') + + info = {'id': video_id, + 'url': video_url, + 'title': video_title, + 'ext': 'flv', + 'format': 'flv', + 'player_url': embed_page_url} + + return [info] + +class EightTracksIE(InfoExtractor): + IE_NAME = '8tracks' + _VALID_URL = r'https?://8tracks.com/(?P[^/]+)/(?P[^/#]+)(?:#.*)?$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + playlist_id = mobj.group('id') + + webpage = self._download_webpage(url, playlist_id) + + m = re.search(r"PAGE.mix = (.*?);\n", webpage, flags=re.DOTALL) + if not m: + raise ExtractorError(u'Cannot find trax information') + json_like = m.group(1) + data = json.loads(json_like) + + session = str(random.randint(0, 1000000000)) + mix_id = data['id'] + track_count = data['tracks_count'] + first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id) + next_url = first_url + res = [] + for i in itertools.count(): + api_json = self._download_webpage(next_url, playlist_id, + note=u'Downloading song information %s/%s' % (str(i+1), track_count), + errnote=u'Failed to download song information') + api_data = json.loads(api_json) + track_data = api_data[u'set']['track'] + info = { + 'id': track_data['id'], + 'url': track_data['track_file_stream_url'], + 'title': track_data['performer'] + u' - ' + track_data['name'], + 'raw_title': track_data['name'], + 'uploader_id': data['user']['login'], + 'ext': 'm4a', + } + res.append(info) + if api_data['set']['at_last_track']: + break + next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id']) + return res + +class KeekIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P\w+)' + IE_NAME = u'keek' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('videoID') + video_url = u'http://cdn.keek.com/keek/video/%s' % video_id + thumbnail = u'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id + webpage = self._download_webpage(url, video_id) + m = re.search(r'[\s\n]+

(?P\w+)

', webpage) + uploader = unescapeHTML(m.group('uploader')) + info = { + 'id':video_id, + 'url':video_url, + 'ext': 'mp4', + 'title': title, + 'thumbnail': thumbnail, + 'uploader': uploader + } + return [info] + +class TEDIE(InfoExtractor): + _VALID_URL=r'''http://www.ted.com/ + ( + ((?Pplaylists)/(?P\d+)) # We have a playlist + | + ((?Ptalks)) # We have a simple talk + ) + /(?P\w+) # Here goes the name and then ".html" + ''' + + def suitable(self, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(self._VALID_URL, url, re.VERBOSE) is not None + + def _real_extract(self, url): + m=re.match(self._VALID_URL, url, re.VERBOSE) + if m.group('type_talk'): + return [self._talk_info(url)] + else : + playlist_id=m.group('playlist_id') + name=m.group('name') + self._downloader.to_screen(u'[%s] Getting info of playlist %s: "%s"' % (self.IE_NAME,playlist_id,name)) + return self._playlist_videos_info(url,name,playlist_id) + + def _talk_video_link(self,mediaSlug): + '''Returns the video link for that mediaSlug''' + return 'http://download.ted.com/talks/%s.mp4' % mediaSlug + + def _playlist_videos_info(self,url,name,playlist_id=0): + '''Returns the videos of the playlist''' + video_RE=r''' + (?P.+?)

' + webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage') + m_videos=re.finditer(video_RE,webpage,re.VERBOSE) + m_names=re.finditer(video_name_RE,webpage) + info=[] + for m_video, m_name in zip(m_videos,m_names): + video_dic={ + 'id': m_video.group('video_id'), + 'url': self._talk_video_link(m_video.group('mediaSlug')), + 'ext': 'mp4', + 'title': m_name.group('fullname') + } + info.append(video_dic) + return info + def _talk_info(self, url, video_id=0): + """Return the video for the talk in the url""" + m=re.match(self._VALID_URL, url,re.VERBOSE) + videoName=m.group('name') + webpage=self._download_webpage(url, video_id, 'Downloading \"%s\" page' % videoName) + # If the url includes the language we get the title translated + title_RE=r'

(?P[\s\w:/\.\?=\+-\\\']*)</span></h1>' + title=re.search(title_RE, webpage).group('title') + info_RE=r'''<script\ type="text/javascript">var\ talkDetails\ =(.*?) + "id":(?P<videoID>[\d]+).*? + "mediaSlug":"(?P<mediaSlug>[\w\d]+?)"''' + info_match=re.search(info_RE,webpage,re.VERBOSE) + video_id=info_match.group('videoID') + mediaSlug=info_match.group('mediaSlug') + video_url=self._talk_video_link(mediaSlug) + info = { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': title + } + return info + +class MySpassIE(InfoExtractor): + _VALID_URL = r'http://www.myspass.de/.*' + + def _real_extract(self, url): + META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s' + + # video id is the last path element of the URL + # usually there is a trailing slash, so also try the second but last + url_path = compat_urllib_parse_urlparse(url).path + url_parent_path, video_id = os.path.split(url_path) + if not video_id: + _, video_id = os.path.split(url_parent_path) + + # get metadata + metadata_url = META_DATA_URL_TEMPLATE % video_id + metadata_text = self._download_webpage(metadata_url, video_id) + metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8')) + + # extract values from metadata + url_flv_el = metadata.find('url_flv') + if url_flv_el is None: + self._downloader.trouble(u'ERROR: unable to extract download url') + return + video_url = url_flv_el.text + extension = os.path.splitext(video_url)[1][1:] + title_el = metadata.find('title') + if title_el is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + title = title_el.text + format_id_el = metadata.find('format_id') + if format_id_el is None: + format = ext + else: + format = format_id_el.text + description_el = metadata.find('description') + if description_el is not None: + description = description_el.text + else: + description = None + imagePreview_el = metadata.find('imagePreview') + if imagePreview_el is not None: + thumbnail = imagePreview_el.text + else: + thumbnail = None + info = { + 'id': video_id, + 'url': video_url, + 'title': title, + 'ext': extension, + 'format': format, + 'thumbnail': thumbnail, + 'description': description + } + return [info] + +def gen_extractors(): + """ Return a list of an instance of every supported extractor. + The order does matter; the first extractor matched is the one handling the URL. + """ + return [ + YoutubePlaylistIE(), + YoutubeChannelIE(), + YoutubeUserIE(), + YoutubeSearchIE(), + YoutubeIE(), + MetacafeIE(), + DailymotionIE(), + GoogleSearchIE(), + PhotobucketIE(), + YahooIE(), + YahooSearchIE(), + DepositFilesIE(), + FacebookIE(), + BlipTVUserIE(), + BlipTVIE(), + VimeoIE(), + MyVideoIE(), + ComedyCentralIE(), + EscapistIE(), + CollegeHumorIE(), + XVideosIE(), + SoundcloudIE(), + InfoQIE(), + MixcloudIE(), + StanfordOpenClassroomIE(), + MTVIE(), + YoukuIE(), + XNXXIE(), + YouJizzIE(), + PornotubeIE(), + YouPornIE(), + GooglePlusIE(), + ArteTvIE(), + NBAIE(), + JustinTVIE(), + FunnyOrDieIE(), + TweetReelIE(), + SteamIE(), + UstreamIE(), + RBMARadioIE(), + EightTracksIE(), + KeekIE(), + TEDIE(), + MySpassIE(), + GenericIE() + ] + +