# Various small unit tests
import io
+import json
import xml.etree.ElementTree
#from youtube_dl.utils import htmlentity_transform
urlencode_postdata,
xpath_with_ns,
parse_iso8601,
+ strip_jsonp,
)
if sys.version_info < (3, 0):
self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
+ def test_strip_jsonp(self):
+ stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);')
+ d = json.loads(stripped)
+ self.assertEqual(d, [{"id": "532cb", "x": 3}])
+
+
if __name__ == '__main__':
unittest.main()
+from __future__ import unicode_literals
+
import re
import time
import xml.etree.ElementTree
from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+ ExtractorError,
+ parse_duration,
+)
class ClipfishIE(InfoExtractor):
- IE_NAME = u'clipfish'
+ IE_NAME = 'clipfish'
_VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/'
_TEST = {
- u'url': u'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
- u'file': u'3966754.mp4',
- u'md5': u'2521cd644e862936cf2e698206e47385',
- u'info_dict': {
- u'title': u'FIFA 14 - E3 2013 Trailer',
- u'duration': 82,
+ 'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
+ 'md5': '2521cd644e862936cf2e698206e47385',
+ 'info_dict': {
+ 'id': '3966754',
+ 'ext': 'mp4',
+ 'title': 'FIFA 14 - E3 2013 Trailer',
+ 'duration': 82,
},
u'skip': 'Blocked in the US'
}
video_url = doc.find('filename').text
if video_url is None:
xml_bytes = xml.etree.ElementTree.tostring(doc)
- raise ExtractorError(u'Cannot find video URL in document %r' %
+ raise ExtractorError('Cannot find video URL in document %r' %
xml_bytes)
thumbnail = doc.find('imageurl').text
- duration_str = doc.find('duration').text
- m = re.match(
- r'^(?P<hours>[0-9]+):(?P<minutes>[0-9]{2}):(?P<seconds>[0-9]{2}):(?P<ms>[0-9]*)$',
- duration_str)
- if m:
- duration = (
- (int(m.group('hours')) * 60 * 60) +
- (int(m.group('minutes')) * 60) +
- (int(m.group('seconds')))
- )
- else:
- duration = None
+ duration = parse_duration(doc.find('duration').text)
return {
'id': video_id,
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ strip_jsonp,
+)
+
+
+class WashingtonPostIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
+ _TEST = {
+ 'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
+ 'playlist': [{
+ 'md5': 'c3f4b4922ffa259243f68e928db2db8c',
+ 'info_dict': {
+ 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
+ 'ext': 'mp4',
+ 'title': 'Breaking Points: The Paper Mine',
+ 'duration': 1287,
+ 'description': 'Overly complicated paper pushing is nothing new to government bureaucracy. But the way federal retirement applications are filed may be the most outdated. David Fahrenthold explains.',
+ 'uploader': 'The Washington Post',
+ 'timestamp': 1395527908,
+ 'upload_date': '20140322',
+ },
+ }, {
+ 'md5': 'f645a07652c2950cd9134bb852c5f5eb',
+ 'info_dict': {
+ 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
+ 'ext': 'mp4',
+ 'title': 'The town bureaucracy sustains',
+ 'description': 'Underneath the friendly town of Boyers is a sea of government paperwork. In a disused limestone mine, hundreds of locals now track, file and process retirement applications for the federal government. We set out to find out what it\'s like to do paperwork 230 feet underground.',
+ 'duration': 2217,
+ 'timestamp': 1395528005,
+ 'upload_date': '20140322',
+ 'uploader': 'The Washington Post',
+ },
+ }]
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ page_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, page_id)
+ title = self._og_search_title(webpage)
+ uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
+ entries = []
+ for i, uuid in enumerate(uuids, start=1):
+ vinfo_all = self._download_json(
+ 'http://www.washingtonpost.com/posttv/c/videojson/%s?resType=jsonp' % uuid,
+ page_id,
+ transform_source=strip_jsonp,
+ note='Downloading information of video %d/%d' % (i, len(uuids))
+ )
+ vinfo = vinfo_all[0]['contentConfig']
+ uploader = vinfo.get('credits', {}).get('source')
+ timestamp = int_or_none(
+ vinfo.get('dateConfig', {}).get('dateFirstPublished'), 1000)
+
+ formats = [{
+ 'format_id': (
+ '%s-%s-%s' % (s.get('type'), s.get('width'), s.get('bitrate'))
+ if s.get('width')
+ else s.get('type')),
+ 'vbr': s.get('bitrate') if s.get('width') != 0 else None,
+ 'width': s.get('width'),
+ 'height': s.get('height'),
+ 'acodec': s.get('audioCodec'),
+ 'vcodec': s.get('videoCodec') if s.get('width') != 0 else 'none',
+ 'filesize': s.get('fileSize'),
+ 'url': s.get('url'),
+ 'ext': 'mp4',
+ 'protocol': {
+ 'MP4': 'http',
+ 'F4F': 'f4m',
+ }.get(s.get('type'))
+ } for s in vinfo.get('streams', [])]
+ source_media_url = vinfo.get('sourceMediaURL')
+ if source_media_url:
+ formats.append({
+ 'format_id': 'source_media',
+ 'url': source_media_url,
+ })
+ self._sort_formats(formats)
+ entries.append({
+ 'id': uuid,
+ 'title': vinfo['title'],
+ 'description': vinfo.get('blurb'),
+ 'uploader': uploader,
+ 'formats': formats,
+ 'duration': int_or_none(vinfo.get('videoDuration'), 100),
+ 'timestamp': timestamp,
+ })
+
+ return {
+ '_type': 'playlist',
+ 'entries': entries,
+ 'id': page_id,
+ 'title': title,
+ }
projects / youtube-dl / commitdiff