rename _parse_mpd to _parse_mpd_formats and add default value for mpd namespace

[youtube-dl] / youtube_dl / extractor / facebook.py

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index ec699ba5405426b07a169b3a6cfdb3a0aeb355d0..ed237f081e2397fe30dc9e4e9c32d590170692de 100644 (file)
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -6,9 +6,11 @@ import socket
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
     error_to_compat_str,
@@ -23,19 +25,30 @@ from ..utils import (
 
 class FacebookIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:\w+\.)?facebook\.com/
-        (?:[^#]*?\#!/)?
-        (?:
-            (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
-            (?:v|video_id)=|
-            [^/]+/videos/(?:[^/]+/)?
-        )
-        (?P<id>[0-9]+)
-        (?:.*)'''
+                (?:
+                    https?://
+                        (?:\w+\.)?facebook\.com/
+                        (?:[^#]*?\#!/)?
+                        (?:
+                            (?:
+                                video/video\.php|
+                                photo\.php|
+                                video\.php|
+                                video/embed
+                            )\?(?:.*?)(?:v|video_id)=|
+                            [^/]+/videos/(?:[^/]+/)?
+                        )|
+                    facebook:
+                )
+                (?P<id>[0-9]+)
+                '''
     _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
     _NETRC_MACHINE = 'facebook'
     IE_NAME = 'facebook'
+
+    _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
+
     _TESTS = [{
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
         'md5': '6a40d33c0eccbb1af76cf0485a052659',
@@ -57,6 +70,16 @@ class FacebookIE(InfoExtractor):
         'expected_warnings': [
             'title'
         ]
+    }, {
+        'note': 'Video with DASH manifest',
+        'url': 'https://www.facebook.com/video.php?v=957955867617029',
+        'md5': '54706e4db4f5ad58fbad82dde1f1213f',
+        'info_dict': {
+            'id': '957955867617029',
+            'ext': 'mp4',
+            'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
+            'uploader': 'Demy de Zeeuw',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -66,6 +89,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
         'only_matching': True,
+    }, {
+        'url': 'facebook:544765982287235',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -136,13 +162,36 @@ class FacebookIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
-        webpage = self._download_webpage(url, video_id)
+        req = sanitized_Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
+        req.add_header('User-Agent', self._CHROME_USER_AGENT)
+        webpage = self._download_webpage(req, video_id)
+
+        video_data = None
 
         BEFORE = '{swf.addParam(param[0], param[1]);});\n'
         AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
         m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
-        if not m:
+        if m:
+            data = dict(json.loads(m.group(1)))
+            params_raw = compat_urllib_parse_unquote(data['params'])
+            video_data = json.loads(params_raw)['video_data']
+
+        def video_data_list2dict(video_data):
+            ret = {}
+            for item in video_data:
+                format_id = item['stream_type']
+                ret.setdefault(format_id, []).append(item)
+            return ret
+
+        if not video_data:
+            server_js_data = self._parse_json(self._search_regex(
+                r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
+            for item in server_js_data['instances']:
+                if item[1][0] == 'VideoConfig':
+                    video_data = video_data_list2dict(item[2][0]['videoData'])
+                    break
+
+        if not video_data:
             m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
             if m_msg is not None:
                 raise ExtractorError(
@@ -150,12 +199,9 @@ class FacebookIE(InfoExtractor):
                     expected=True)
             else:
                 raise ExtractorError('Cannot parse data')
-        data = dict(json.loads(m.group(1)))
-        params_raw = compat_urllib_parse_unquote(data['params'])
-        params = json.loads(params_raw)
 
         formats = []
-        for format_id, f in params['video_data'].items():
+        for format_id, f in video_data.items():
             if not f or not isinstance(f, list):
                 continue
             for quality in ('sd', 'hd'):
@@ -167,9 +213,15 @@ class FacebookIE(InfoExtractor):
                             'url': src,
                             'preference': -10 if format_id == 'progressive' else 0,
                         })
+            dash_manifest = f[0].get('dash_manifest')
+            if dash_manifest:
+                formats.extend(self._parse_mpd_formats(
+                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
+        self._sort_formats(formats)
+
         video_title = self._html_search_regex(
             r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
             default=None)
@@ -188,3 +240,33 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'uploader': uploader,
         }
+
+
+class FacebookPostIE(InfoExtractor):
+    IE_NAME = 'facebook:post'
+    _VALID_URL = r'https?://(?:\w+\.)?facebook\.com/[^/]+/posts/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
+        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
+        'info_dict': {
+            'id': '544765982287235',
+            'ext': 'mp4',
+            'title': '"What are you doing running in the snow?"',
+            'uploader': 'FailArmy',
+        }
+    }
+
+    def _real_extract(self, url):
+        post_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, post_id)
+
+        entries = [
+            self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
+            for video_id in self._parse_json(
+                self._search_regex(
+                    r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
+                    webpage, 'video ids', group='ids'),
+                post_id)]
+
+        return self.playlist_result(entries, post_id)