Merge remote-tracking branch 'origin/vimeo_passworded_videos'

[youtube-dl] / youtube_dl / InfoExtractors.py

diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index ed2beac783eb83ca41dc843af648ee471f598488..574d417beecc97718edcc6da864acb4d503d325a 100755 (executable)
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -699,14 +699,14 @@ class YoutubeIE(InfoExtractor):
                         pass
                     else:
                         # We report the original error
-                        self._downloader.report_error(sub_error)
+                        self._downloader.report_warning(sub_error)
 
         if self._downloader.params.get('allsubtitles', False):
             video_subtitles = self._extract_all_subtitles(video_id)
             for video_subtitle in video_subtitles:
                 (sub_error, sub_lang, sub) = video_subtitle
                 if sub_error:
-                    self._downloader.report_error(sub_error)
+                    self._downloader.report_warning(sub_error)
 
         if self._downloader.params.get('listsubtitles', False):
             sub_lang_list = self._list_available_subtitles(video_id)
@@ -1096,6 +1096,25 @@ class VimeoIE(InfoExtractor):
     _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|player)\.)?vimeo(?P<pro>pro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)'
     IE_NAME = u'vimeo'
 
+    def _verify_video_password(self, url, video_id, webpage):
+        password = self._downloader.params.get('password', None)
+        if password is None:
+            raise ExtractorError(u'This video is protected by a password, use the --password option')
+        token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1)
+        data = compat_urllib_parse.urlencode({'password': password,
+                                              'token': token})
+        # I didn't manage to use the password with https
+        if url.startswith('https'):
+            pass_url = url.replace('https','http')
+        else:
+            pass_url = url
+        password_request = compat_urllib_request.Request(pass_url+'/password', data)
+        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Cookie', 'xsrft=%s' % token)
+        pass_web = self._download_webpage(password_request, video_id,
+                                          u'Verifying the password',
+                                          u'Wrong password')
+
     def _real_extract(self, url, new_video=True):
         # Extract ID from URL
         mobj = re.match(self._VALID_URL, url)
@@ -1124,6 +1143,10 @@ class VimeoIE(InfoExtractor):
         except:
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
                 raise ExtractorError(u'The author has restricted the access to this video, try with the "--referer" option')
+
+            if re.search('If so please provide the correct password.', webpage):
+                self._verify_video_password(url, video_id, webpage)
+                return self._real_extract(url)
             else:
                 raise ExtractorError(u'Unable to extract info section')
 
@@ -1409,6 +1432,13 @@ class GenericIE(InfoExtractor):
         if mobj is None:
             # Try to find twitter cards info
             mobj = re.search(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
+        if mobj is None:
+            # We look for Open Graph info:
+            # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
+            m_video_type = re.search(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
+            # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
+            if m_video_type is not None:
+                mobj = re.search(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)
         if mobj is None:
             raise ExtractorError(u'Invalid URL: %s' % url)
 
@@ -1609,9 +1639,10 @@ class YoutubePlaylistIE(InfoExtractor):
                 # Number of videos is a multiple of self._MAX_RESULTS
                 break
 
-            videos += [ (entry['yt$position']['$t'], entry['content']['src'])
-                        for entry in response['feed']['entry']
-                        if 'content' in entry ]
+            for entry in response['feed']['entry']:
+                index = entry['yt$position']['$t']
+                if 'media$group' in entry and 'media$player' in entry['media$group']:
+                    videos.append((index, entry['media$group']['media$player']['url']))
 
             if len(response['feed']['entry']) < self._MAX_RESULTS:
                 break
@@ -4544,6 +4575,37 @@ class GametrailersIE(InfoExtractor):
                 'description': video_description,
                 }
 
+class StatigramIE(InfoExtractor):
+    _VALID_URL = r'(?:http://)?(?:www\.)?statigr\.am/p/([^/]+)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group(1)
+        webpage = self._download_webpage(url, video_id)
+        video_url = self._html_search_regex(
+            r'<meta property="og:video:secure_url" content="(.+?)">',
+            webpage, u'video URL')
+        thumbnail_url = self._html_search_regex(
+            r'<meta property="og:image" content="(.+?)" />',
+            webpage, u'thumbnail URL', fatal=False)
+        html_title = self._html_search_regex(
+            r'<title>(.+?)</title>',
+            webpage, u'title')
+        title = html_title.rpartition(u' | Statigram')[0]
+        uploader = self._html_search_regex(
+            r'@(.+) \(Videos\)', title, u'uploader name', fatal=False)
+        ext = 'mp4'
+
+        return [{
+            'id':        video_id,
+            'url':       video_url,
+            'ext':       ext,
+            'title':     title,
+            'thumbnail': thumbnail_url,
+            'uploader' : uploader
+        }]
+
 def gen_extractors():
     """ Return a list of an instance of every supported extractor.
     The order does matter; the first extractor matched is the one handling the URL.
@@ -4610,6 +4672,7 @@ def gen_extractors():
         HypemIE(),
         Vbox7IE(),
         GametrailersIE(),
+        StatigramIE(),
         GenericIE()
     ]