[myspass] Move into own file and default to mp4 ext
[youtube-dl] / youtube_dl / InfoExtractors.py
index d2b80c5580338401b7d6540bfd0fde191dd7044a..c3d0e619130c05dc7cc863ec45f2f184c8c03fa7 100755 (executable)
@@ -25,6 +25,7 @@ from .extractor.comedycentral import ComedyCentralIE
 from .extractor.collegehumor import CollegeHumorIE
 from .extractor.dailymotion import DailymotionIE
 from .extractor.depositfiles import DepositFilesIE
 from .extractor.collegehumor import CollegeHumorIE
 from .extractor.dailymotion import DailymotionIE
 from .extractor.depositfiles import DepositFilesIE
+from .extractor.eighttracks import EightTracksIE
 from .extractor.escapist import EscapistIE
 from .extractor.facebook import FacebookIE
 from .extractor.funnyordie import FunnyOrDieIE
 from .extractor.escapist import EscapistIE
 from .extractor.facebook import FacebookIE
 from .extractor.funnyordie import FunnyOrDieIE
@@ -34,13 +35,16 @@ from .extractor.googleplus import GooglePlusIE
 from .extractor.googlesearch import GoogleSearchIE
 from .extractor.infoq import InfoQIE
 from .extractor.justintv import JustinTVIE
 from .extractor.googlesearch import GoogleSearchIE
 from .extractor.infoq import InfoQIE
 from .extractor.justintv import JustinTVIE
+from .extractor.keek import KeekIE
 from .extractor.metacafe import MetacafeIE
 from .extractor.mixcloud import MixcloudIE
 from .extractor.mtv import MTVIE
 from .extractor.metacafe import MetacafeIE
 from .extractor.mixcloud import MixcloudIE
 from .extractor.mtv import MTVIE
+from .extractor.myspass import MySpassIE
 from .extractor.myvideo import MyVideoIE
 from .extractor.nba import NBAIE
 from .extractor.statigram import StatigramIE
 from .extractor.photobucket import PhotobucketIE
 from .extractor.myvideo import MyVideoIE
 from .extractor.nba import NBAIE
 from .extractor.statigram import StatigramIE
 from .extractor.photobucket import PhotobucketIE
+from .extractor.pornotube import PornotubeIE
 from .extractor.rbmaradio import RBMARadioIE
 from .extractor.soundcloud import SoundcloudIE, SoundcloudSetIE
 from .extractor.stanfordoc import StanfordOpenClassroomIE
 from .extractor.rbmaradio import RBMARadioIE
 from .extractor.soundcloud import SoundcloudIE, SoundcloudSetIE
 from .extractor.stanfordoc import StanfordOpenClassroomIE
@@ -52,7 +56,9 @@ from .extractor.worldstarhiphop import WorldStarHipHopIE
 from .extractor.xnxx import XNXXIE
 from .extractor.xvideos import XVideosIE
 from .extractor.yahoo import YahooIE, YahooSearchIE
 from .extractor.xnxx import XNXXIE
 from .extractor.xvideos import XVideosIE
 from .extractor.yahoo import YahooIE, YahooSearchIE
+from .extractor.youjizz import YouJizzIE
 from .extractor.youku import YoukuIE
 from .extractor.youku import YoukuIE
+from .extractor.youporn import YouPornIE
 from .extractor.youtube import YoutubeIE, YoutubePlaylistIE, YoutubeSearchIE, YoutubeUserIE, YoutubeChannelIE
 from .extractor.zdf import ZDFIE
 
 from .extractor.youtube import YoutubeIE, YoutubePlaylistIE, YoutubeSearchIE, YoutubeUserIE, YoutubeChannelIE
 from .extractor.zdf import ZDFIE
 
@@ -75,312 +81,14 @@ from .extractor.zdf import ZDFIE
 
 
 
 
 
 
-class YouPornIE(InfoExtractor):
-    """Information extractor for youporn.com."""
-    _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+)'
 
 
-    def _print_formats(self, formats):
-        """Print all available formats"""
-        print(u'Available formats:')
-        print(u'ext\t\tformat')
-        print(u'---------------------------------')
-        for format in formats:
-            print(u'%s\t\t%s'  % (format['ext'], format['format']))
 
 
-    def _specific(self, req_format, formats):
-        for x in formats:
-            if(x["format"]==req_format):
-                return x
-        return None
 
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-        video_id = mobj.group('videoid')
-
-        req = compat_urllib_request.Request(url)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
-
-        # Get JSON parameters
-        json_params = self._search_regex(r'var currentVideo = new Video\((.*)\);', webpage, u'JSON parameters')
-        try:
-            params = json.loads(json_params)
-        except:
-            raise ExtractorError(u'Invalid JSON')
-
-        self.report_extraction(video_id)
-        try:
-            video_title = params['title']
-            upload_date = unified_strdate(params['release_date_f'])
-            video_description = params['description']
-            video_uploader = params['submitted_by']
-            thumbnail = params['thumbnails'][0]['image']
-        except KeyError:
-            raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
-
-        # Get all of the formats available
-        DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
-        download_list_html = self._search_regex(DOWNLOAD_LIST_RE,
-            webpage, u'download list').strip()
-
-        # Get all of the links from the page
-        LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">'
-        links = re.findall(LINK_RE, download_list_html)
-        if(len(links) == 0):
-            raise ExtractorError(u'ERROR: no known formats available for video')
-
-        self.to_screen(u'Links found: %d' % len(links))
-
-        formats = []
-        for link in links:
-
-            # A link looks like this:
-            # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
-            # A path looks like this:
-            # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
-            video_url = unescapeHTML( link )
-            path = compat_urllib_parse_urlparse( video_url ).path
-            extension = os.path.splitext( path )[1][1:]
-            format = path.split('/')[4].split('_')[:2]
-            size = format[0]
-            bitrate = format[1]
-            format = "-".join( format )
-            # title = u'%s-%s-%s' % (video_title, size, bitrate)
-
-            formats.append({
-                'id': video_id,
-                'url': video_url,
-                'uploader': video_uploader,
-                'upload_date': upload_date,
-                'title': video_title,
-                'ext': extension,
-                'format': format,
-                'thumbnail': thumbnail,
-                'description': video_description
-            })
-
-        if self._downloader.params.get('listformats', None):
-            self._print_formats(formats)
-            return
-
-        req_format = self._downloader.params.get('format', None)
-        self.to_screen(u'Format: %s' % req_format)
-
-        if req_format is None or req_format == 'best':
-            return [formats[0]]
-        elif req_format == 'worst':
-            return [formats[-1]]
-        elif req_format in ('-1', 'all'):
-            return formats
-        else:
-            format = self._specific( req_format, formats )
-            if result is None:
-                raise ExtractorError(u'Requested format not available')
-            return [format]
-
-
-
-class PornotubeIE(InfoExtractor):
-    """Information extractor for pornotube.com."""
-    _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-
-        video_id = mobj.group('videoid')
-        video_title = mobj.group('title')
-
-        # Get webpage content
-        webpage = self._download_webpage(url, video_id)
-
-        # Get the video URL
-        VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",'
-        video_url = self._search_regex(VIDEO_URL_RE, webpage, u'video url')
-        video_url = compat_urllib_parse.unquote(video_url)
-
-        #Get the uploaded date
-        VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
-        upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False)
-        if upload_date: upload_date = unified_strdate(upload_date)
-
-        info = {'id': video_id,
-                'url': video_url,
-                'uploader': None,
-                'upload_date': upload_date,
-                'title': video_title,
-                'ext': 'flv',
-                'format': 'flv'}
-
-        return [info]
-
-class YouJizzIE(InfoExtractor):
-    """Information extractor for youjizz.com."""
-    _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+).html$'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-
-        video_id = mobj.group('videoid')
-
-        # Get webpage content
-        webpage = self._download_webpage(url, video_id)
-
-        # Get the video title
-        video_title = self._html_search_regex(r'<title>(?P<title>.*)</title>',
-            webpage, u'title').strip()
 
 
-        # Get the embed page
-        result = re.search(r'https?://www.youjizz.com/videos/embed/(?P<videoid>[0-9]+)', webpage)
-        if result is None:
-            raise ExtractorError(u'ERROR: unable to extract embed page')
 
 
-        embed_page_url = result.group(0).strip()
-        video_id = result.group('videoid')
 
 
-        webpage = self._download_webpage(embed_page_url, video_id)
 
 
-        # Get the video URL
-        video_url = self._search_regex(r'so.addVariable\("file",encodeURIComponent\("(?P<source>[^"]+)"\)\);',
-            webpage, u'video URL')
-
-        info = {'id': video_id,
-                'url': video_url,
-                'title': video_title,
-                'ext': 'flv',
-                'format': 'flv',
-                'player_url': embed_page_url}
-
-        return [info]
-
-class EightTracksIE(InfoExtractor):
-    IE_NAME = '8tracks'
-    _VALID_URL = r'https?://8tracks.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-        playlist_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, playlist_id)
-
-        json_like = self._search_regex(r"PAGE.mix = (.*?);\n", webpage, u'trax information', flags=re.DOTALL)
-        data = json.loads(json_like)
-
-        session = str(random.randint(0, 1000000000))
-        mix_id = data['id']
-        track_count = data['tracks_count']
-        first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
-        next_url = first_url
-        res = []
-        for i in itertools.count():
-            api_json = self._download_webpage(next_url, playlist_id,
-                note=u'Downloading song information %s/%s' % (str(i+1), track_count),
-                errnote=u'Failed to download song information')
-            api_data = json.loads(api_json)
-            track_data = api_data[u'set']['track']
-            info = {
-                'id': track_data['id'],
-                'url': track_data['track_file_stream_url'],
-                'title': track_data['performer'] + u' - ' + track_data['name'],
-                'raw_title': track_data['name'],
-                'uploader_id': data['user']['login'],
-                'ext': 'm4a',
-            }
-            res.append(info)
-            if api_data['set']['at_last_track']:
-                break
-            next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id'])
-        return res
-
-class KeekIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<videoID>\w+)'
-    IE_NAME = u'keek'
-
-    def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
-
-        video_url = u'http://cdn.keek.com/keek/video/%s' % video_id
-        thumbnail = u'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id
-        webpage = self._download_webpage(url, video_id)
-
-        video_title = self._html_search_regex(r'<meta property="og:title" content="(?P<title>.*?)"',
-            webpage, u'title')
 
 
-        uploader = self._html_search_regex(r'<div class="user-name-and-bio">[\S\s]+?<h2>(?P<uploader>.+?)</h2>',
-            webpage, u'uploader', fatal=False)
-
-        info = {
-                'id': video_id,
-                'url': video_url,
-                'ext': 'mp4',
-                'title': video_title,
-                'thumbnail': thumbnail,
-                'uploader': uploader
-        }
-        return [info]
-
-
-class MySpassIE(InfoExtractor):
-    _VALID_URL = r'http://www.myspass.de/.*'
-
-    def _real_extract(self, url):
-        META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s'
-
-        # video id is the last path element of the URL
-        # usually there is a trailing slash, so also try the second but last
-        url_path = compat_urllib_parse_urlparse(url).path
-        url_parent_path, video_id = os.path.split(url_path)
-        if not video_id:
-            _, video_id = os.path.split(url_parent_path)
-
-        # get metadata
-        metadata_url = META_DATA_URL_TEMPLATE % video_id
-        metadata_text = self._download_webpage(metadata_url, video_id)
-        metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8'))
-
-        # extract values from metadata
-        url_flv_el = metadata.find('url_flv')
-        if url_flv_el is None:
-            raise ExtractorError(u'Unable to extract download url')
-        video_url = url_flv_el.text
-        extension = os.path.splitext(video_url)[1][1:]
-        title_el = metadata.find('title')
-        if title_el is None:
-            raise ExtractorError(u'Unable to extract title')
-        title = title_el.text
-        format_id_el = metadata.find('format_id')
-        if format_id_el is None:
-            format = ext
-        else:
-            format = format_id_el.text
-        description_el = metadata.find('description')
-        if description_el is not None:
-            description = description_el.text
-        else:
-            description = None
-        imagePreview_el = metadata.find('imagePreview')
-        if imagePreview_el is not None:
-            thumbnail = imagePreview_el.text
-        else:
-            thumbnail = None
-        info = {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'ext': extension,
-            'format': format,
-            'thumbnail': thumbnail,
-            'description': description
-        }
-        return [info]
 
 class SpiegelIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$'
 
 class SpiegelIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$'