X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;ds=sidebyside;f=youtube_dl%2FInfoExtractors.py;h=e5118a717220ee43b037bfcbb158b78e21e3ebe2;hb=2ab1c5ed1a5bf3f63b3e7e6f09d59e431cbe783c;hp=d1b239abfcd6917c14a6eba0e58cbe5f9750e1bf;hpb=59ae15a507cbf93de13f8fda8444d2d9a251747a;p=youtube-dl diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py old mode 100644 new mode 100755 index d1b239abf..e5118a717 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import absolute_import + import datetime import netrc import os @@ -12,7 +14,7 @@ import xml.etree.ElementTree import random import math -from utils import * +from .utils import * class InfoExtractor(object): @@ -484,8 +486,8 @@ class YoutubeIE(InfoExtractor): # Extension video_extension = self._video_extensions.get(format_param, 'flv') - video_format = '{} - {}'.format(format_param if format_param else video_extension, - self._video_dimensions.get(format_param, '???')) + video_format = '{0} - {1}'.format(format_param if format_param else video_extension, + self._video_dimensions.get(format_param, '???')) results.append({ 'id': video_id, @@ -664,7 +666,8 @@ class DailymotionIE(InfoExtractor): request.add_header('Cookie', 'family_filter=off') try: self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() + webpage_bytes = compat_urllib_request.urlopen(request).read() + webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) return @@ -699,7 +702,7 @@ class DailymotionIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: unable to extract title') return - video_title = unescapeHTML(mobj.group('title').decode('utf-8')) + video_title = unescapeHTML(mobj.group('title')) video_uploader = None mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage) @@ -719,12 +722,12 @@ class DailymotionIE(InfoExtractor): video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), + 'id': video_id, + 'url': video_url, + 'uploader': video_uploader, 'upload_date': video_upload_date, 'title': video_title, - 'ext': video_extension.decode('utf-8'), + 'ext': video_extension, }] @@ -1059,7 +1062,8 @@ class VimeoIE(InfoExtractor): request = compat_urllib_request.Request(url, None, std_headers) try: self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() + webpage_bytes = compat_urllib_request.urlopen(request).read() + webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) return @@ -1070,8 +1074,8 @@ class VimeoIE(InfoExtractor): self.report_extraction(video_id) # Extract the config JSON - config = webpage.split(' = {config:')[1].split(',assets:')[0] try: + config = webpage.split(' = {config:')[1].split(',assets:')[0] config = json.loads(config) except: self._downloader.trouble(u'ERROR: unable to extract info section') @@ -1087,7 +1091,7 @@ class VimeoIE(InfoExtractor): video_thumbnail = config["video"]["thumbnail"] # Extract video description - video_description = get_element_by_id("description", webpage.decode('utf8')) + video_description = get_element_by_id("description", webpage) if video_description: video_description = clean_html(video_description) else: video_description = '' @@ -1405,22 +1409,22 @@ class GenericIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: unable to extract title') return - video_title = mobj.group(1).decode('utf-8') + video_title = mobj.group(1) # video uploader is domain name mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract title') return - video_uploader = mobj.group(1).decode('utf-8') + video_uploader = mobj.group(1) return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), + 'id': video_id, + 'url': video_url, 'uploader': video_uploader, 'upload_date': None, 'title': video_title, - 'ext': video_extension.decode('utf-8'), + 'ext': video_extension, }] @@ -1669,10 +1673,10 @@ class YahooSearchIE(InfoExtractor): class YoutubePlaylistIE(InfoExtractor): """Information Extractor for YouTube playlists.""" - _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*' + _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*' _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en' _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s' - _MORE_PAGES_INDICATOR = r'yt-uix-pager-next' + _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" IE_NAME = u'youtube:playlist' def __init__(self, downloader=None): @@ -1711,7 +1715,7 @@ class YoutubePlaylistIE(InfoExtractor): url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum) request = compat_urllib_request.Request(url) try: - page = compat_urllib_request.urlopen(request).read() + page = compat_urllib_request.urlopen(request).read().decode('utf8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) return @@ -1723,10 +1727,12 @@ class YoutubePlaylistIE(InfoExtractor): ids_in_page.append(mobj.group(1)) video_ids.extend(ids_in_page) - if re.search(self._MORE_PAGES_INDICATOR, page) is None: + if self._MORE_PAGES_INDICATOR not in page: break pagenum = pagenum + 1 + total = len(video_ids) + playliststart = self._downloader.params.get('playliststart', 1) - 1 playlistend = self._downloader.params.get('playlistend', -1) if playlistend == -1: @@ -1734,6 +1740,11 @@ class YoutubePlaylistIE(InfoExtractor): else: video_ids = video_ids[playliststart:playlistend] + if len(video_ids) == total: + self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total)) + else: + self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids))) + for id in video_ids: self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) return @@ -1744,7 +1755,7 @@ class YoutubeChannelIE(InfoExtractor): _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$" _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en' - _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO + _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" IE_NAME = u'youtube:channel' def report_download_page(self, channel_id, pagenum): @@ -1768,7 +1779,7 @@ class YoutubeChannelIE(InfoExtractor): url = self._TEMPLATE_URL % (channel_id, pagenum) request = compat_urllib_request.Request(url) try: - page = compat_urllib_request.urlopen(request).read() + page = compat_urllib_request.urlopen(request).read().decode('utf8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) return @@ -1780,10 +1791,12 @@ class YoutubeChannelIE(InfoExtractor): ids_in_page.append(mobj.group(1)) video_ids.extend(ids_in_page) - if re.search(self._MORE_PAGES_INDICATOR, page) is None: + if self._MORE_PAGES_INDICATOR not in page: break pagenum = pagenum + 1 + self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) + for id in video_ids: self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) return @@ -2260,7 +2273,7 @@ class BlipTVIE(InfoExtractor): else: cchar = '?' json_url = url + cchar + 'skin=json&version=2&no_wrap=1' - request = compat_urllib_request.Request(json_url.encode('utf-8')) + request = compat_urllib_request.Request(json_url) self.report_extraction(mobj.group(1)) info = None try: @@ -2285,7 +2298,8 @@ class BlipTVIE(InfoExtractor): return if info is None: # Regular URL try: - json_code = urlh.read() + json_code_bytes = urlh.read() + json_code = json_code_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err)) return @@ -2385,7 +2399,19 @@ class MyVideoIE(InfoExtractor): class ComedyCentralIE(InfoExtractor): """Information extractor for The Daily Show and Colbert Report """ - _VALID_URL = r'^(:(?Ptds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?Pthedailyshow|colbertnation)\.com/full-episodes/(?P.*)$' + # urls can be abbreviations like :thedailyshow or :colbert + # urls for episodes like: + # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day + # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news + # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524 + _VALID_URL = r"""^(:(?Ptds|thedailyshow|cr|colbert|colbertnation|colbertreport) + |(https?://)?(www\.)? + (?Pthedailyshow|colbertnation)\.com/ + (full-episodes/(?P.*)| + (?P + (the-colbert-report-(videos|collections)/(?P[0-9]+)/[^/]*/(?P.*?)) + |(watch/(?P[^/]*)/(?P.*))))) + $""" IE_NAME = u'comedycentral' _available_formats = ['3500', '2200', '1700', '1200', '750', '400'] @@ -2407,6 +2433,10 @@ class ComedyCentralIE(InfoExtractor): '400': '384x216', } + def suitable(self, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(self._VALID_URL, url, re.VERBOSE) is not None + def report_extraction(self, episode_id): self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) @@ -2427,7 +2457,7 @@ class ComedyCentralIE(InfoExtractor): def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) + mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return @@ -2437,14 +2467,21 @@ class ComedyCentralIE(InfoExtractor): url = u'http://www.thedailyshow.com/full-episodes/' else: url = u'http://www.colbertnation.com/full-episodes/' - mobj = re.match(self._VALID_URL, url) + mobj = re.match(self._VALID_URL, url, re.VERBOSE) assert mobj is not None - dlNewest = not mobj.group('episode') - if dlNewest: - epTitle = mobj.group('showname') + if mobj.group('clip'): + if mobj.group('showname') == 'thedailyshow': + epTitle = mobj.group('tdstitle') + else: + epTitle = mobj.group('cntitle') + dlNewest = False else: - epTitle = mobj.group('episode') + dlNewest = not mobj.group('episode') + if dlNewest: + epTitle = mobj.group('showname') + else: + epTitle = mobj.group('episode') req = compat_urllib_request.Request(url) self.report_extraction(epTitle) @@ -2456,7 +2493,7 @@ class ComedyCentralIE(InfoExtractor): return if dlNewest: url = htmlHandle.geturl() - mobj = re.match(self._VALID_URL, url) + mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url) return @@ -2465,14 +2502,14 @@ class ComedyCentralIE(InfoExtractor): return epTitle = mobj.group('episode') - mMovieParams = re.findall('(?:[0-9]+)/(?P.*)$' IE_NAME = u'collegehumor' - def report_webpage(self, video_id): + def report_manifest(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + self._downloader.to_screen(u'[%s] %s: Downloading XML manifest' % (self.IE_NAME, video_id)) def report_extraction(self, video_id): """Report information extraction.""" @@ -2665,29 +2703,14 @@ class CollegeHumorIE(InfoExtractor): return video_id = mobj.group('videoid') - self.report_webpage(video_id) - request = compat_urllib_request.Request(url) - try: - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - return - - m = re.search(r'id="video:(?P[0-9]+)"', webpage) - if m is None: - self._downloader.trouble(u'ERROR: Cannot extract internal video ID') - return - internal_video_id = m.group('internalvideoid') - info = { 'id': video_id, - 'internal_id': internal_video_id, 'uploader': None, 'upload_date': None, } self.report_extraction(video_id) - xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id + xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id try: metaXml = compat_urllib_request.urlopen(xmlUrl).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: @@ -2699,13 +2722,34 @@ class CollegeHumorIE(InfoExtractor): videoNode = mdoc.findall('./video')[0] info['description'] = videoNode.findall('./description')[0].text info['title'] = videoNode.findall('./caption')[0].text - info['url'] = videoNode.findall('./file')[0].text info['thumbnail'] = videoNode.findall('./thumbnail')[0].text - info['ext'] = info['url'].rpartition('.')[2] + manifest_url = videoNode.findall('./file')[0].text except IndexError: self._downloader.trouble(u'\nERROR: Invalid metadata XML file') return + manifest_url += '?hdcore=2.10.3' + self.report_manifest(video_id) + try: + manifestXml = compat_urllib_request.urlopen(manifest_url).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) + return + + adoc = xml.etree.ElementTree.fromstring(manifestXml) + try: + media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0] + node_id = media_node.attrib['url'] + video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text + except IndexError as err: + self._downloader.trouble(u'\nERROR: Invalid manifest file') + return + + url_pr = compat_urllib_parse_urlparse(manifest_url) + url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1' + + info['url'] = url + info['ext'] = 'f4f' return [info] @@ -2728,13 +2772,14 @@ class XVideosIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return - video_id = mobj.group(1).decode('utf-8') + video_id = mobj.group(1) self.report_webpage(video_id) request = compat_urllib_request.Request(r'http://www.xvideos.com/video' + video_id) try: - webpage = compat_urllib_request.urlopen(request).read() + webpage_bytes = compat_urllib_request.urlopen(request).read() + webpage = webpage_bytes.decode('utf-8', 'replace') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) return @@ -2747,7 +2792,7 @@ class XVideosIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video url') return - video_url = compat_urllib_parse.unquote(mobj.group(1).decode('utf-8')) + video_url = compat_urllib_parse.unquote(mobj.group(1)) # Extract title @@ -2755,7 +2800,7 @@ class XVideosIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video title') return - video_title = mobj.group(1).decode('utf-8') + video_title = mobj.group(1) # Extract video thumbnail @@ -2763,7 +2808,7 @@ class XVideosIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video thumbnail') return - video_thumbnail = mobj.group(0).decode('utf-8') + video_thumbnail = mobj.group(0) info = { 'id': video_id, @@ -2794,13 +2839,13 @@ class SoundcloudIE(InfoExtractor): def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) - def report_webpage(self, video_id): + def report_resolve(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id)) def report_extraction(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id)) def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -2809,65 +2854,47 @@ class SoundcloudIE(InfoExtractor): return # extract uploader (which is in the url) - uploader = mobj.group(1).decode('utf-8') + uploader = mobj.group(1) # extract simple title (uploader + slug of song title) - slug_title = mobj.group(2).decode('utf-8') + slug_title = mobj.group(2) simple_title = uploader + u'-' + slug_title - self.report_webpage('%s/%s' % (uploader, slug_title)) + self.report_resolve('%s/%s' % (uploader, slug_title)) - request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title)) + url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title) + resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28' + request = compat_urllib_request.Request(resolv_url) try: - webpage = compat_urllib_request.urlopen(request).read() + info_json_bytes = compat_urllib_request.urlopen(request).read() + info_json = info_json_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) return + info = json.loads(info_json) + video_id = info['id'] self.report_extraction('%s/%s' % (uploader, slug_title)) - # extract uid and stream token that soundcloud hands out for access - mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage) - if mobj: - video_id = mobj.group(1) - stream_token = mobj.group(2) - - # extract unsimplified title - mobj = re.search('"title":"(.*?)",', webpage) - if mobj: - title = mobj.group(1).decode('utf-8') - else: - title = simple_title - - # construct media url (with uid/token) - mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s" - mediaURL = mediaURL % (video_id, stream_token) - - # description - description = u'No description available' - mobj = re.search('track-description-value">

(.*?)

', webpage) - if mobj: - description = mobj.group(1) - - # upload date - upload_date = None - mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)", webpage) - if mobj: - try: - upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d') - except Exception as err: - self._downloader.to_stderr(compat_str(err)) + streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28' + request = compat_urllib_request.Request(streams_url) + try: + stream_json_bytes = compat_urllib_request.urlopen(request).read() + stream_json = stream_json_bytes.decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) + return - # for soundcloud, a request to a cross domain is required for cookies - request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers) + streams = json.loads(stream_json) + mediaURL = streams['http_mp3_128_url'] return [{ - 'id': video_id.decode('utf-8'), + 'id': info['id'], 'url': mediaURL, - 'uploader': uploader.decode('utf-8'), - 'upload_date': upload_date, - 'title': title, + 'uploader': info['user']['username'], + 'upload_date': info['created_at'], + 'title': info['title'], 'ext': u'mp3', - 'description': description.decode('utf-8') + 'description': info['description'], }] @@ -3324,7 +3351,8 @@ class YoukuIE(InfoExtractor): self.report_extraction(video_id) try: - config = json.loads(jsondata) + jsonstr = jsondata.decode('utf-8') + config = json.loads(jsonstr) video_title = config['data'][0]['title'] seed = config['data'][0]['seed'] @@ -3347,15 +3375,8 @@ class YoukuIE(InfoExtractor): fileid = config['data'][0]['streamfileids'][format] - seg_number = len(config['data'][0]['segs'][format]) - - keys=[] - for i in xrange(seg_number): - keys.append(config['data'][0]['segs'][format][i]['k']) - - #TODO check error - #youku only could be viewed from mainland china - except: + keys = [s['k'] for s in config['data'][0]['segs'][format]] + except (UnicodeDecodeError, ValueError, KeyError): self._downloader.trouble(u'ERROR: unable to extract info section') return @@ -3405,13 +3426,14 @@ class XNXXIE(InfoExtractor): if mobj is None: self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return - video_id = mobj.group(1).decode('utf-8') + video_id = mobj.group(1) self.report_webpage(video_id) # Get webpage content try: - webpage = compat_urllib_request.urlopen(url).read() + webpage_bytes = compat_urllib_request.urlopen(url).read() + webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err) return @@ -3420,19 +3442,19 @@ class XNXXIE(InfoExtractor): if result is None: self._downloader.trouble(u'ERROR: unable to extract video url') return - video_url = compat_urllib_parse.unquote(result.group(1).decode('utf-8')) + video_url = compat_urllib_parse.unquote(result.group(1)) result = re.search(self.VIDEO_TITLE_RE, webpage) if result is None: self._downloader.trouble(u'ERROR: unable to extract video title') return - video_title = result.group(1).decode('utf-8') + video_title = result.group(1) result = re.search(self.VIDEO_THUMB_RE, webpage) if result is None: self._downloader.trouble(u'ERROR: unable to extract video thumbnail') return - video_thumbnail = result.group(1).decode('utf-8') + video_thumbnail = result.group(1) return [{ 'id': video_id, @@ -3566,3 +3588,126 @@ class GooglePlusIE(InfoExtractor): 'title': video_title.decode('utf-8'), 'ext': video_extension.decode('utf-8'), }] + +class NBAIE(InfoExtractor): + _VALID_URL = r'^(?:https?://)?(?:watch\.|www\.)?nba\.com/(?:nba/)?video(/[^?]*)(\?.*)?$' + IE_NAME = u'nba' + + def report_extraction(self, video_id): + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group(1) + if video_id.endswith('/index.html'): + video_id = video_id[:-len('/index.html')] + + self.report_extraction(video_id) + try: + urlh = compat_urllib_request.urlopen(url) + webpage_bytes = urlh.read() + webpage = webpage_bytes.decode('utf-8', 'ignore') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) + return + + video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4' + def _findProp(rexp, default=None): + m = re.search(rexp, webpage) + if m: + return unescapeHTML(m.group(1)) + else: + return default + + shortened_video_id = video_id.rpartition('/')[2] + title = _findProp(r'Date: (.*?)'), + 'description': _findProp(r'
(.*?)'), + } + return [info] + +class JustinTVIE(InfoExtractor): + """Information extractor for justin.tv and twitch.tv""" + # TODO: One broadcast may be split into multiple videos. The key + # 'broadcast_id' is the same for all parts, and 'broadcast_part' + # starts at 1 and increases. Can we treat all parts as one video? + +# _VALID_URL = r"""^(?:http(?:s?)://)?www\.(?:justin|twitch)\.tv/ +# ([^/]+)(?:/b/([^/]+))?/?(?:#.*)?$""" + _VALID_URL = r'^http://www.twitch.tv/(.*)$' + IE_NAME = u'justin.tv' + + _max_justin_results = 1000 + _justin_page_limit = 100 + + def report_extraction(self, file_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id)) + + # Return count of items, list of *valid* items + def _parse_page(self, url): + print url + try: + urlh = compat_urllib_request.urlopen(url) + webpage_bytes = urlh.read() + webpage = webpage_bytes.decode('utf-8', 'ignore') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.trouble(u'ERROR: unable to download video info JSON: %s' % compat_str(err)) + return + + response = json.loads(webpage) + info = [] + for clip in response: + video_url = clip['video_file_url'] + if video_url: + video_extension = os.path.splitext(video_url)[1][1:] + video_date = re.sub('-', '', clip['created_on'][:10]) + info.append({ + 'id': clip['id'], + 'url': video_url, + 'title': clip['title'], + 'uploader': clip['user_id'] or clip['channel_id'], + 'upload_date': video_date, + 'ext': video_extension, + }) + print len(response) + return (len(response), info) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + api = 'http://api.justin.tv' + video_id = mobj.group(mobj.lastindex) + paged = False + if mobj.lastindex == 1: + paged = True + api += '/channel/archives/%s.json' + else: + api += '/clip/show/%s.json' + api = api % (video_id,) + + self.report_extraction(video_id) + + info = [] + offset = 0 + limit = self._justin_page_limit + while offset < self._max_justin_results: + page_url = api + ('?offset=%d&limit=%d' % (offset, limit)) + page_count, page_info = self._parse_page(page_url) + info.extend(page_info) + if not paged or page_count != limit: + break + offset += limit + return info