X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;ds=sidebyside;f=youtube_dl%2FInfoExtractors.py;h=c3d0e619130c05dc7cc863ec45f2f184c8c03fa7;hb=97d2db017c67bedd1e50d149b51d60fcbfd26041;hp=d2b80c5580338401b7d6540bfd0fde191dd7044a;hpb=e10e576fed8264d52b186adfeb4eb53183a75959;p=youtube-dl diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index d2b80c558..c3d0e6191 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -25,6 +25,7 @@ from .extractor.comedycentral import ComedyCentralIE from .extractor.collegehumor import CollegeHumorIE from .extractor.dailymotion import DailymotionIE from .extractor.depositfiles import DepositFilesIE +from .extractor.eighttracks import EightTracksIE from .extractor.escapist import EscapistIE from .extractor.facebook import FacebookIE from .extractor.funnyordie import FunnyOrDieIE @@ -34,13 +35,16 @@ from .extractor.googleplus import GooglePlusIE from .extractor.googlesearch import GoogleSearchIE from .extractor.infoq import InfoQIE from .extractor.justintv import JustinTVIE +from .extractor.keek import KeekIE from .extractor.metacafe import MetacafeIE from .extractor.mixcloud import MixcloudIE from .extractor.mtv import MTVIE +from .extractor.myspass import MySpassIE from .extractor.myvideo import MyVideoIE from .extractor.nba import NBAIE from .extractor.statigram import StatigramIE from .extractor.photobucket import PhotobucketIE +from .extractor.pornotube import PornotubeIE from .extractor.rbmaradio import RBMARadioIE from .extractor.soundcloud import SoundcloudIE, SoundcloudSetIE from .extractor.stanfordoc import StanfordOpenClassroomIE @@ -52,7 +56,9 @@ from .extractor.worldstarhiphop import WorldStarHipHopIE from .extractor.xnxx import XNXXIE from .extractor.xvideos import XVideosIE from .extractor.yahoo import YahooIE, YahooSearchIE +from .extractor.youjizz import YouJizzIE from .extractor.youku import YoukuIE +from .extractor.youporn import YouPornIE from .extractor.youtube import YoutubeIE, YoutubePlaylistIE, YoutubeSearchIE, YoutubeUserIE, YoutubeChannelIE from .extractor.zdf import ZDFIE @@ -75,312 +81,14 @@ from .extractor.zdf import ZDFIE -class YouPornIE(InfoExtractor): - """Information extractor for youporn.com.""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P[0-9]+)/(?P[^/]+)' - def _print_formats(self, formats): - """Print all available formats""" - print(u'Available formats:') - print(u'ext\t\tformat') - print(u'---------------------------------') - for format in formats: - print(u'%s\t\t%s' % (format['ext'], format['format'])) - def _specific(self, req_format, formats): - for x in formats: - if(x["format"]==req_format): - return x - return None - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group('videoid') - - req = compat_urllib_request.Request(url) - req.add_header('Cookie', 'age_verified=1') - webpage = self._download_webpage(req, video_id) - - # Get JSON parameters - json_params = self._search_regex(r'var currentVideo = new Video\((.*)\);', webpage, u'JSON parameters') - try: - params = json.loads(json_params) - except: - raise ExtractorError(u'Invalid JSON') - - self.report_extraction(video_id) - try: - video_title = params['title'] - upload_date = unified_strdate(params['release_date_f']) - video_description = params['description'] - video_uploader = params['submitted_by'] - thumbnail = params['thumbnails'][0]['image'] - except KeyError: - raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1]) - - # Get all of the formats available - DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>' - download_list_html = self._search_regex(DOWNLOAD_LIST_RE, - webpage, u'download list').strip() - - # Get all of the links from the page - LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">' - links = re.findall(LINK_RE, download_list_html) - if(len(links) == 0): - raise ExtractorError(u'ERROR: no known formats available for video') - - self.to_screen(u'Links found: %d' % len(links)) - - formats = [] - for link in links: - - # A link looks like this: - # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0 - # A path looks like this: - # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4 - video_url = unescapeHTML( link ) - path = compat_urllib_parse_urlparse( video_url ).path - extension = os.path.splitext( path )[1][1:] - format = path.split('/')[4].split('_')[:2] - size = format[0] - bitrate = format[1] - format = "-".join( format ) - # title = u'%s-%s-%s' % (video_title, size, bitrate) - - formats.append({ - 'id': video_id, - 'url': video_url, - 'uploader': video_uploader, - 'upload_date': upload_date, - 'title': video_title, - 'ext': extension, - 'format': format, - 'thumbnail': thumbnail, - 'description': video_description - }) - - if self._downloader.params.get('listformats', None): - self._print_formats(formats) - return - - req_format = self._downloader.params.get('format', None) - self.to_screen(u'Format: %s' % req_format) - - if req_format is None or req_format == 'best': - return [formats[0]] - elif req_format == 'worst': - return [formats[-1]] - elif req_format in ('-1', 'all'): - return formats - else: - format = self._specific( req_format, formats ) - if result is None: - raise ExtractorError(u'Requested format not available') - return [format] - - - -class PornotubeIE(InfoExtractor): - """Information extractor for pornotube.com.""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('videoid') - video_title = mobj.group('title') - - # Get webpage content - webpage = self._download_webpage(url, video_id) - - # Get the video URL - VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",' - video_url = self._search_regex(VIDEO_URL_RE, webpage, u'video url') - video_url = compat_urllib_parse.unquote(video_url) - - #Get the uploaded date - VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by' - upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False) - if upload_date: upload_date = unified_strdate(upload_date) - - info = {'id': video_id, - 'url': video_url, - 'uploader': None, - 'upload_date': upload_date, - 'title': video_title, - 'ext': 'flv', - 'format': 'flv'} - - return [info] - -class YouJizzIE(InfoExtractor): - """Information extractor for youjizz.com.""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+).html$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('videoid') - - # Get webpage content - webpage = self._download_webpage(url, video_id) - - # Get the video title - video_title = self._html_search_regex(r'<title>(?P<title>.*)', - webpage, u'title').strip() - # Get the embed page - result = re.search(r'https?://www.youjizz.com/videos/embed/(?P[0-9]+)', webpage) - if result is None: - raise ExtractorError(u'ERROR: unable to extract embed page') - embed_page_url = result.group(0).strip() - video_id = result.group('videoid') - webpage = self._download_webpage(embed_page_url, video_id) - # Get the video URL - video_url = self._search_regex(r'so.addVariable\("file",encodeURIComponent\("(?P[^"]+)"\)\);', - webpage, u'video URL') - - info = {'id': video_id, - 'url': video_url, - 'title': video_title, - 'ext': 'flv', - 'format': 'flv', - 'player_url': embed_page_url} - - return [info] - -class EightTracksIE(InfoExtractor): - IE_NAME = '8tracks' - _VALID_URL = r'https?://8tracks.com/(?P[^/]+)/(?P[^/#]+)(?:#.*)?$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - playlist_id = mobj.group('id') - - webpage = self._download_webpage(url, playlist_id) - - json_like = self._search_regex(r"PAGE.mix = (.*?);\n", webpage, u'trax information', flags=re.DOTALL) - data = json.loads(json_like) - - session = str(random.randint(0, 1000000000)) - mix_id = data['id'] - track_count = data['tracks_count'] - first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id) - next_url = first_url - res = [] - for i in itertools.count(): - api_json = self._download_webpage(next_url, playlist_id, - note=u'Downloading song information %s/%s' % (str(i+1), track_count), - errnote=u'Failed to download song information') - api_data = json.loads(api_json) - track_data = api_data[u'set']['track'] - info = { - 'id': track_data['id'], - 'url': track_data['track_file_stream_url'], - 'title': track_data['performer'] + u' - ' + track_data['name'], - 'raw_title': track_data['name'], - 'uploader_id': data['user']['login'], - 'ext': 'm4a', - } - res.append(info) - if api_data['set']['at_last_track']: - break - next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id']) - return res - -class KeekIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P\w+)' - IE_NAME = u'keek' - - def _real_extract(self, url): - m = re.match(self._VALID_URL, url) - video_id = m.group('videoID') - - video_url = u'http://cdn.keek.com/keek/video/%s' % video_id - thumbnail = u'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id - webpage = self._download_webpage(url, video_id) - - video_title = self._html_search_regex(r'[\S\s]+?

(?P.+?)

', - webpage, u'uploader', fatal=False) - - info = { - 'id': video_id, - 'url': video_url, - 'ext': 'mp4', - 'title': video_title, - 'thumbnail': thumbnail, - 'uploader': uploader - } - return [info] - - -class MySpassIE(InfoExtractor): - _VALID_URL = r'http://www.myspass.de/.*' - - def _real_extract(self, url): - META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s' - - # video id is the last path element of the URL - # usually there is a trailing slash, so also try the second but last - url_path = compat_urllib_parse_urlparse(url).path - url_parent_path, video_id = os.path.split(url_path) - if not video_id: - _, video_id = os.path.split(url_parent_path) - - # get metadata - metadata_url = META_DATA_URL_TEMPLATE % video_id - metadata_text = self._download_webpage(metadata_url, video_id) - metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8')) - - # extract values from metadata - url_flv_el = metadata.find('url_flv') - if url_flv_el is None: - raise ExtractorError(u'Unable to extract download url') - video_url = url_flv_el.text - extension = os.path.splitext(video_url)[1][1:] - title_el = metadata.find('title') - if title_el is None: - raise ExtractorError(u'Unable to extract title') - title = title_el.text - format_id_el = metadata.find('format_id') - if format_id_el is None: - format = ext - else: - format = format_id_el.text - description_el = metadata.find('description') - if description_el is not None: - description = description_el.text - else: - description = None - imagePreview_el = metadata.find('imagePreview') - if imagePreview_el is not None: - thumbnail = imagePreview_el.text - else: - thumbnail = None - info = { - 'id': video_id, - 'url': video_url, - 'title': title, - 'ext': extension, - 'format': format, - 'thumbnail': thumbnail, - 'description': description - } - return [info] class SpiegelIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P[0-9]+)(?:\.html)?(?:#.*)?$'