X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;ds=sidebyside;f=youtube_dl%2Fextractor%2Fpbs.py;h=09aef7fb9f455d8d3247350b26a6d66043877605;hb=542130a5d914e5f02acb872c88866194c66a612d;hp=6e2ef0fba4ff883216ef77927da29a8f072a2d2f;hpb=3cddb8d6a776b09afd7f50772fa30cb536b1149a;p=youtube-dl

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6e2ef0fba..09aef7fb9 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -4,13 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
     js_to_json,
     strip_jsonp,
+    strip_or_none,
     unified_strdate,
     US_RATINGS,
 )
@@ -201,7 +201,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365006249',
                 'ext': 'mp4',
                 'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
-                'description': 'md5:36f341ae62e251b8f5bd2b754b95a071',
+                'description': 'md5:31b664af3c65fd07fa460d306b837d00',
                 'duration': 3190,
             },
         },
@@ -212,7 +212,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365297690',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - Losing Iraq',
-                'description': 'md5:4d3eaa01f94e61b3e73704735f1196d9',
+                'description': 'md5:5979a4d069b157f622d02bff62fbe654',
                 'duration': 5050,
             },
         },
@@ -223,7 +223,7 @@ class PBSIE(InfoExtractor):
                 'id': '2201174722',
                 'ext': 'mp4',
                 'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
-                'description': 'md5:95a19f568689d09a166dff9edada3301',
+                'description': 'md5:86ab9a3d04458b876147b355788b8781',
                 'duration': 801,
             },
         },
@@ -268,7 +268,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'player',
                 'ext': 'mp4',
                 'title': 'American Experience - Death and the Civil War, Chapter 1',
-                'description': 'md5:1b80a74e0380ed2a4fb335026de1600d',
+                'description': 'md5:67fa89a9402e2ee7d08f53b920674c18',
                 'duration': 682,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -294,13 +294,13 @@ class PBSIE(InfoExtractor):
             # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
             # https://github.com/rg3/youtube-dl/issues/7059)
             'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
-            'md5': '84ced42850d78f1d4650297356e95e6f',
+            'md5': '59b0ef5009f9ac8a319cc5efebcd865e',
             'info_dict': {
                 'id': '2365546844',
                 'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
                 'ext': 'mp4',
                 'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
-                'description': 'md5:54033c6baa1f9623607c6e2ed245888b',
+                'description': 'md5:c0ff7475a4b70261c7e58f493c2792a5',
                 'duration': 1480,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -313,7 +313,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'the-atomic-artists',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - The Atomic Artists',
-                'description': 'md5:1a2481e86b32b2e12ec1905dd473e2c1',
+                'description': 'md5:f677e4520cfacb4a5ce1471e31b57800',
                 'duration': 723,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -324,7 +324,7 @@ class PBSIE(InfoExtractor):
         {
             # Serves hd only via wigget/partnerplayer page
             'url': 'http://www.pbs.org/video/2365641075/',
-            'md5': 'acfd4c400b48149a44861cb16dd305cf',
+            'md5': 'fdf907851eab57211dd589cf12006666',
             'info_dict': {
                 'id': '2365641075',
                 'ext': 'mp4',
@@ -353,11 +353,16 @@ class PBSIE(InfoExtractor):
     def _extract_webpage(self, url):
         mobj = re.match(self._VALID_URL, url)
 
+        description = None
+
         presumptive_id = mobj.group('presumptive_id')
         display_id = presumptive_id
         if presumptive_id:
             webpage = self._download_webpage(url, display_id)
 
+            description = strip_or_none(self._og_search_description(
+                webpage, default=None) or self._html_search_meta(
+                'description', webpage, default=None))
             upload_date = unified_strdate(self._search_regex(
                 r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
                 webpage, 'upload date', default=None))
@@ -370,7 +375,7 @@ class PBSIE(InfoExtractor):
             for p in MULTI_PART_REGEXES:
                 tabbed_videos = re.findall(p, webpage)
                 if tabbed_videos:
-                    return tabbed_videos, presumptive_id, upload_date
+                    return tabbed_videos, presumptive_id, upload_date, description
 
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed
@@ -382,7 +387,7 @@ class PBSIE(InfoExtractor):
             media_id = self._search_regex(
                 MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
             if media_id:
-                return media_id, presumptive_id, upload_date
+                return media_id, presumptive_id, upload_date, description
 
             # Fronline video embedded via flp
             video_id = self._search_regex(
@@ -399,7 +404,7 @@ class PBSIE(InfoExtractor):
                     'http://www.pbs.org/wgbh/pages/frontline/.json/getdir/getdir%d.json' % prg_id,
                     presumptive_id, 'Downloading getdir JSON',
                     transform_source=strip_jsonp)
-                return getdir['mid'], presumptive_id, upload_date
+                return getdir['mid'], presumptive_id, upload_date, description
 
             for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
                 url = self._search_regex(
@@ -423,10 +428,10 @@ class PBSIE(InfoExtractor):
             video_id = mobj.group('id')
             display_id = video_id
 
-        return video_id, display_id, None
+        return video_id, display_id, None, description
 
     def _real_extract(self, url):
-        video_id, display_id, upload_date = self._extract_webpage(url)
+        video_id, display_id, upload_date, description = self._extract_webpage(url)
 
         if isinstance(video_id, list):
             entries = [self.url_result(
@@ -501,14 +506,18 @@ class PBSIE(InfoExtractor):
         if http_url:
             for m3u8_format in m3u8_formats:
                 bitrate = self._search_regex(r'(\d+)k', m3u8_format['url'], 'bitrate', default=None)
-                # extract only the formats that we know that they will be available as http format.
+                # lower qualities(150k and 192k) are not available as http formats
+                # https://github.com/rg3/youtube-dl/commit/cbc032c8b70a038a69259378c92b4ba97b42d491#commitcomment-17313656
+                # we will try to extract any http format higher than than the lowest quality documented in
                 # https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
+                # as there also undocumented http formats formats(4500k and 6500k)
+                # http://www.pbs.org/video/2365815229/
                 if not bitrate or int(bitrate) < 400:
                     continue
                 f_url = re.sub(r'\d+k|baseline', bitrate + 'k', http_url)
                 # This may produce invalid links sometimes (e.g.
                 # http://www.pbs.org/wgbh/frontline/film/suicide-plan)
-                if not self._is_valid_url(f_url, display_id, 'http-%s video' % bitrate):
+                if not self._is_valid_url(f_url, display_id, 'http-%sk video' % bitrate):
                     continue
                 f = m3u8_format.copy()
                 f.update({
@@ -551,11 +560,14 @@ class PBSIE(InfoExtractor):
         if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-:]+', '', info['title'])
 
+        description = info.get('description') or info.get(
+            'program', {}).get('description') or description
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'description': info.get('description') or info.get('program', {}).get('description'),
+            'description': description,
             'thumbnail': info.get('image_url'),
             'duration': int_or_none(info.get('duration')),
             'age_limit': age_limit,