Merge branch 'jython-support'

[youtube-dl] / youtube_dl / extractor / facebook.py

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 899b0896b92b5d4ddfc7f291f6b064334f3830af..6c6c3b1bd460407322aab1f35ddd8e55cefaad17 100644 (file)
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -6,9 +6,11 @@ import socket
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
     error_to_compat_str,
@@ -32,8 +34,9 @@ class FacebookIE(InfoExtractor):
                                 video/video\.php|
                                 photo\.php|
                                 video\.php|
-                                video/embed
-                            )\?(?:.*?)(?:v|video_id)=|
+                                video/embed|
+                                story\.php
+                            )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?
                         )|
                     facebook:
@@ -44,6 +47,9 @@ class FacebookIE(InfoExtractor):
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
     _NETRC_MACHINE = 'facebook'
     IE_NAME = 'facebook'
+
+    _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
+
     _TESTS = [{
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
         'md5': '6a40d33c0eccbb1af76cf0485a052659',
@@ -65,6 +71,16 @@ class FacebookIE(InfoExtractor):
         'expected_warnings': [
             'title'
         ]
+    }, {
+        'note': 'Video with DASH manifest',
+        'url': 'https://www.facebook.com/video.php?v=957955867617029',
+        'md5': '54706e4db4f5ad58fbad82dde1f1213f',
+        'info_dict': {
+            'id': '957955867617029',
+            'ext': 'mp4',
+            'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
+            'uploader': 'Demy de Zeeuw',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -77,6 +93,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'facebook:544765982287235',
         'only_matching': True,
+    }, {
+        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -147,8 +166,9 @@ class FacebookIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
-        webpage = self._download_webpage(url, video_id)
+        req = sanitized_Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
+        req.add_header('User-Agent', self._CHROME_USER_AGENT)
+        webpage = self._download_webpage(req, video_id)
 
         video_data = None
 
@@ -170,7 +190,7 @@ class FacebookIE(InfoExtractor):
         if not video_data:
             server_js_data = self._parse_json(self._search_regex(
                 r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
-            for item in server_js_data['instances']:
+            for item in server_js_data.get('instances', []):
                 if item[1][0] == 'VideoConfig':
                     video_data = video_data_list2dict(item[2][0]['videoData'])
                     break
@@ -192,14 +212,23 @@ class FacebookIE(InfoExtractor):
                 for src_type in ('src', 'src_no_ratelimit'):
                     src = f[0].get('%s_%s' % (quality, src_type))
                     if src:
+                        preference = -10 if format_id == 'progressive' else 0
+                        if quality == 'hd':
+                            preference += 5
                         formats.append({
                             'format_id': '%s_%s_%s' % (format_id, quality, src_type),
                             'url': src,
-                            'preference': -10 if format_id == 'progressive' else 0,
+                            'preference': preference,
                         })
+            dash_manifest = f[0].get('dash_manifest')
+            if dash_manifest:
+                formats.extend(self._parse_mpd_formats(
+                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
+        self._sort_formats(formats)
+
         video_title = self._html_search_regex(
             r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
             default=None)