X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;ds=sidebyside;f=youtube_dl%2FInfoExtractors.py;h=b53071c43373b31492fb79d0a479b47af59f8578;hb=993693aa796fa2d50e0ce3cc7b0d551ffcc4a786;hp=9cfff153b12fce1c4db4c1420b6f0caab323a8c2;hpb=95eb771dcda47f948b050da85c7ff22539e3ee12;p=youtube-dl diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py old mode 100644 new mode 100755 index 9cfff153b..b53071c43 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -272,7 +272,7 @@ class YoutubeIE(InfoExtractor): request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form)) try: self.report_login() - login_results = compat_urllib_request.urlopen(request).read() + login_results = compat_urllib_request.urlopen(request).read().decode('utf-8') if re.search(r'(?i)]* name="loginForm"', login_results) is not None: self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') return @@ -288,7 +288,7 @@ class YoutubeIE(InfoExtractor): request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) try: self.report_age_confirmation() - age_results = compat_urllib_request.urlopen(request).read() + age_results = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) return @@ -399,7 +399,7 @@ class YoutubeIE(InfoExtractor): self.report_video_subtitles_download(video_id) request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) try: - srt_list = compat_urllib_request.urlopen(request).read() + srt_list = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list) @@ -416,14 +416,14 @@ class YoutubeIE(InfoExtractor): raise Trouble(u'WARNING: no closed captions found in the specified language') request = compat_urllib_request.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id)) try: - srt_xml = compat_urllib_request.urlopen(request).read() + srt_xml = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) if not srt_xml: raise Trouble(u'WARNING: unable to download video subtitles') - video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8')) + video_subtitles = self._closed_captions_xml_to_srt(srt_xml) except Trouble as trouble: - self._downloader.trouble(trouble[0]) + self._downloader.trouble(str(trouble)) if 'length_seconds' not in video_info: self._downloader.trouble(u'WARNING: unable to extract video duration') @@ -666,7 +666,8 @@ class DailymotionIE(InfoExtractor): request.add_header('Cookie', 'family_filter=off') try: self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() + webpage_bytes = compat_urllib_request.urlopen(request).read() + webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) return @@ -701,7 +702,7 @@ class DailymotionIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: unable to extract title') return - video_title = unescapeHTML(mobj.group('title').decode('utf-8')) + video_title = unescapeHTML(mobj.group('title')) video_uploader = None mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage) @@ -721,105 +722,12 @@ class DailymotionIE(InfoExtractor): video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), + 'id': video_id, + 'url': video_url, + 'uploader': video_uploader, 'upload_date': video_upload_date, 'title': video_title, - 'ext': video_extension.decode('utf-8'), - }] - - -class GoogleIE(InfoExtractor): - """Information extractor for video.google.com.""" - - _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*' - IE_NAME = u'video.google' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id) - - def _real_extract(self, url): - # Extract id from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - return - - video_id = mobj.group(1) - - video_extension = 'mp4' - - # Retrieve video webpage to extract further information - request = compat_urllib_request.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id) - try: - self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) - return - - # Extract URL, uploader, and title from webpage - self.report_extraction(video_id) - mobj = re.search(r"download_url:'([^']+)'", webpage) - if mobj is None: - video_extension = 'flv' - mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - mediaURL = compat_urllib_parse.unquote(mobj.group(1)) - mediaURL = mediaURL.replace('\\x3d', '\x3d') - mediaURL = mediaURL.replace('\\x26', '\x26') - - video_url = mediaURL - - mobj = re.search(r'(.*)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - video_title = mobj.group(1).decode('utf-8') - - # Extract video description - mobj = re.search(r'([^<]*)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video description') - return - video_description = mobj.group(1).decode('utf-8') - if not video_description: - video_description = 'No description available.' - - # Extract video thumbnail - if self._downloader.params.get('forcethumbnail', False): - request = compat_urllib_request.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id))) - try: - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) - return - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') - return - video_thumbnail = mobj.group(1) - else: # we need something to pass to process_info - video_thumbnail = '' - - return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': None, - 'upload_date': None, - 'title': video_title, - 'ext': video_extension.decode('utf-8'), + 'ext': video_extension, }] @@ -891,6 +799,7 @@ class PhotobucketIE(InfoExtractor): class YahooIE(InfoExtractor): """Information extractor for video.yahoo.com.""" + _WORKING = False # _VALID_URL matches all Yahoo! Video URLs # _VPAGE_URL matches only the extractable '/watch/' URLs _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?' @@ -1061,7 +970,8 @@ class VimeoIE(InfoExtractor): request = compat_urllib_request.Request(url, None, std_headers) try: self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() + webpage_bytes = compat_urllib_request.urlopen(request).read() + webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) return @@ -1089,7 +999,7 @@ class VimeoIE(InfoExtractor): video_thumbnail = config["video"]["thumbnail"] # Extract video description - video_description = get_element_by_id("description", webpage.decode('utf8')) + video_description = get_element_by_id("description", webpage) if video_description: video_description = clean_html(video_description) else: video_description = '' @@ -1261,7 +1171,7 @@ class ArteTvIE(InfoExtractor): 'url': compat_urllib_parse.unquote(info.get('url')), 'uploader': u'arte.tv', 'upload_date': info.get('date'), - 'title': info.get('title'), + 'title': info.get('title').decode('utf-8'), 'ext': u'mp4', 'format': u'NA', 'player_url': None, @@ -1407,22 +1317,22 @@ class GenericIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: unable to extract title') return - video_title = mobj.group(1).decode('utf-8') + video_title = mobj.group(1) # video uploader is domain name mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract title') return - video_uploader = mobj.group(1).decode('utf-8') + video_uploader = mobj.group(1) return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), + 'id': video_id, + 'url': video_url, 'uploader': video_uploader, 'upload_date': None, 'title': video_title, - 'ext': video_extension.decode('utf-8'), + 'ext': video_extension, }] @@ -1586,6 +1496,8 @@ class GoogleSearchIE(InfoExtractor): class YahooSearchIE(InfoExtractor): """Information Extractor for Yahoo! Video search queries.""" + + _WORKING = False _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+' _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s' _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"' @@ -1674,7 +1586,7 @@ class YoutubePlaylistIE(InfoExtractor): _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*' _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en' _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s' - _MORE_PAGES_INDICATOR = r'yt-uix-pager-next' + _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" IE_NAME = u'youtube:playlist' def __init__(self, downloader=None): @@ -1713,7 +1625,7 @@ class YoutubePlaylistIE(InfoExtractor): url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum) request = compat_urllib_request.Request(url) try: - page = compat_urllib_request.urlopen(request).read() + page = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) return @@ -1725,10 +1637,12 @@ class YoutubePlaylistIE(InfoExtractor): ids_in_page.append(mobj.group(1)) video_ids.extend(ids_in_page) - if re.search(self._MORE_PAGES_INDICATOR, page) is None: + if self._MORE_PAGES_INDICATOR not in page: break pagenum = pagenum + 1 + total = len(video_ids) + playliststart = self._downloader.params.get('playliststart', 1) - 1 playlistend = self._downloader.params.get('playlistend', -1) if playlistend == -1: @@ -1736,6 +1650,11 @@ class YoutubePlaylistIE(InfoExtractor): else: video_ids = video_ids[playliststart:playlistend] + if len(video_ids) == total: + self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total)) + else: + self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids))) + for id in video_ids: self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) return @@ -1746,7 +1665,7 @@ class YoutubeChannelIE(InfoExtractor): _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$" _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en' - _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO + _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" IE_NAME = u'youtube:channel' def report_download_page(self, channel_id, pagenum): @@ -1770,7 +1689,7 @@ class YoutubeChannelIE(InfoExtractor): url = self._TEMPLATE_URL % (channel_id, pagenum) request = compat_urllib_request.Request(url) try: - page = compat_urllib_request.urlopen(request).read() + page = compat_urllib_request.urlopen(request).read().decode('utf8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) return @@ -1782,10 +1701,12 @@ class YoutubeChannelIE(InfoExtractor): ids_in_page.append(mobj.group(1)) video_ids.extend(ids_in_page) - if re.search(self._MORE_PAGES_INDICATOR, page) is None: + if self._MORE_PAGES_INDICATOR not in page: break pagenum = pagenum + 1 + self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) + for id in video_ids: self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) return @@ -1833,7 +1754,7 @@ class YoutubeUserIE(InfoExtractor): request = compat_urllib_request.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)) try: - page = compat_urllib_request.urlopen(request).read() + page = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) return @@ -2262,7 +2183,7 @@ class BlipTVIE(InfoExtractor): else: cchar = '?' json_url = url + cchar + 'skin=json&version=2&no_wrap=1' - request = compat_urllib_request.Request(json_url.encode('utf-8')) + request = compat_urllib_request.Request(json_url) self.report_extraction(mobj.group(1)) info = None try: @@ -2287,7 +2208,8 @@ class BlipTVIE(InfoExtractor): return if info is None: # Regular URL try: - json_code = urlh.read() + json_code_bytes = urlh.read() + json_code = json_code_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err)) return @@ -2355,7 +2277,7 @@ class MyVideoIE(InfoExtractor): request = compat_urllib_request.Request('http://www.myvideo.de/watch/%s' % video_id) try: self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() + webpage = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) return @@ -2387,7 +2309,19 @@ class MyVideoIE(InfoExtractor): class ComedyCentralIE(InfoExtractor): """Information extractor for The Daily Show and Colbert Report """ - _VALID_URL = r'^(:(?Ptds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?Pthedailyshow|colbertnation)\.com/full-episodes/(?P.*)$' + # urls can be abbreviations like :thedailyshow or :colbert + # urls for episodes like: + # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day + # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news + # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524 + _VALID_URL = r"""^(:(?Ptds|thedailyshow|cr|colbert|colbertnation|colbertreport) + |(https?://)?(www\.)? + (?Pthedailyshow|colbertnation)\.com/ + (full-episodes/(?P.*)| + (?P + (the-colbert-report-(videos|collections)/(?P[0-9]+)/[^/]*/(?P.*?)) + |(watch/(?P[^/]*)/(?P.*))))) + $""" IE_NAME = u'comedycentral' _available_formats = ['3500', '2200', '1700', '1200', '750', '400'] @@ -2409,6 +2343,10 @@ class ComedyCentralIE(InfoExtractor): '400': '384x216', } + def suitable(self, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(self._VALID_URL, url, re.VERBOSE) is not None + def report_extraction(self, episode_id): self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) @@ -2429,7 +2367,7 @@ class ComedyCentralIE(InfoExtractor): def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) + mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return @@ -2439,14 +2377,21 @@ class ComedyCentralIE(InfoExtractor): url = u'http://www.thedailyshow.com/full-episodes/' else: url = u'http://www.colbertnation.com/full-episodes/' - mobj = re.match(self._VALID_URL, url) + mobj = re.match(self._VALID_URL, url, re.VERBOSE) assert mobj is not None - dlNewest = not mobj.group('episode') - if dlNewest: - epTitle = mobj.group('showname') + if mobj.group('clip'): + if mobj.group('showname') == 'thedailyshow': + epTitle = mobj.group('tdstitle') + else: + epTitle = mobj.group('cntitle') + dlNewest = False else: - epTitle = mobj.group('episode') + dlNewest = not mobj.group('episode') + if dlNewest: + epTitle = mobj.group('showname') + else: + epTitle = mobj.group('episode') req = compat_urllib_request.Request(url) self.report_extraction(epTitle) @@ -2458,7 +2403,7 @@ class ComedyCentralIE(InfoExtractor): return if dlNewest: url = htmlHandle.geturl() - mobj = re.match(self._VALID_URL, url) + mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url) return @@ -2467,14 +2412,14 @@ class ComedyCentralIE(InfoExtractor): return epTitle = mobj.group('episode') - mMovieParams = re.findall('(?:Date: (.*?)'), + 'description': _findProp(r'
(.*?)'), + } + return [info] + +class JustinTVIE(InfoExtractor): + """Information extractor for justin.tv and twitch.tv""" + # TODO: One broadcast may be split into multiple videos. The key + # 'broadcast_id' is the same for all parts, and 'broadcast_part' + # starts at 1 and increases. Can we treat all parts as one video? + + _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?(?:twitch|justin)\.tv/ + ([^/]+)(?:/b/([^/]+))?/?(?:\#.*)?$""" + _JUSTIN_PAGE_LIMIT = 100 + IE_NAME = u'justin.tv' + + def report_extraction(self, file_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id)) + + def report_download_page(self, channel, offset): + """Report attempt to download a single page of videos.""" + self._downloader.to_screen(u'[%s] %s: Downloading video information from %d to %d' % + (self.IE_NAME, channel, offset, offset + self._JUSTIN_PAGE_LIMIT)) + + # Return count of items, list of *valid* items + def _parse_page(self, url): + try: + urlh = compat_urllib_request.urlopen(url) + webpage_bytes = urlh.read() + webpage = webpage_bytes.decode('utf-8', 'ignore') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.trouble(u'ERROR: unable to download video info JSON: %s' % compat_str(err)) + return + + response = json.loads(webpage) + info = [] + for clip in response: + video_url = clip['video_file_url'] + if video_url: + video_extension = os.path.splitext(video_url)[1][1:] + video_date = re.sub('-', '', clip['created_on'][:10]) + info.append({ + 'id': clip['id'], + 'url': video_url, + 'title': clip['title'], + 'uploader': clip.get('user_id', clip.get('channel_id')), + 'upload_date': video_date, + 'ext': video_extension, + }) + return (len(response), info) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + api = 'http://api.justin.tv' + video_id = mobj.group(mobj.lastindex) + paged = False + if mobj.lastindex == 1: + paged = True + api += '/channel/archives/%s.json' + else: + api += '/clip/show/%s.json' + api = api % (video_id,) + + self.report_extraction(video_id) + + info = [] + offset = 0 + limit = self._JUSTIN_PAGE_LIMIT + while True: + if paged: + self.report_download_page(video_id, offset) + page_url = api + ('?offset=%d&limit=%d' % (offset, limit)) + page_count, page_info = self._parse_page(page_url) + info.extend(page_info) + if not paged or page_count != limit: + break + offset += limit + return info