# Author: Gergely Imreh
# Author: Philipp Hagemeister <phihag@phihag.de>
# License: Public domain code
-from __future__ import with_statement
-import contextlib
import cookielib
-import ctypes
import datetime
-import email.utils
import gzip
import htmlentitydefs
import httplib
import warnings
import zlib
+if os.name == 'nt':
+ import ctypes
+
+try:
+ import email.utils
+except ImportError: # Python 2.4
+ import email.Utils
try:
import cStringIO as StringIO
except ImportError:
try:
import json
-except ImportError: # Python <2.5, use trivialjson (https://github.com/phihag/trivialjson):
+except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
import re
class json(object):
@staticmethod
nopart: Do not use temporary .part files.
updatetime: Use the Last-modified header to set output file timestamps.
writedescription: Write the video description to a .description file
+ writeinfojson: Write the video description to a .info.json file
"""
params = None
pass
def report_writedescription(self, descfn):
- """ Report that the description file has been written """
- self.to_screen(u'[info] Video description written to: %s' % descfn, ignore_encoding_errors=True)
+ """ Report that the description file is being written """
+ self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True)
+
+ def report_writeinfojson(self, infofn):
+ """ Report that the metadata file has been written """
+ self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True)
def report_destination(self, filename):
"""Report destination filename."""
if self.params.get('writedescription', False):
try:
descfn = filename + '.description'
- with contextlib.closing(open(descfn, 'wb')) as descfile:
- descfile.write(info_dict['description'].encode('utf-8'))
self.report_writedescription(descfn)
+ descfile = open(descfn, 'wb')
+ try:
+ descfile.write(info_dict['description'].encode('utf-8'))
+ finally:
+ descfile.close()
except (OSError, IOError):
self.trouble(u'ERROR: Cannot write description file: %s' % str(descfn))
return
+ if self.params.get('writeinfojson', False):
+ infofn = filename + '.info.json'
+ self.report_writeinfojson(infofn)
+ try:
+ json.dump
+ except (NameError,AttributeError):
+ self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
+ return
+ try:
+ infof = open(infofn, 'wb')
+ try:
+ json.dump(info_dict, infof)
+ finally:
+ infof.close()
+ except (OSError, IOError):
+ self.trouble(u'ERROR: Cannot write metadata to JSON file: %s' % str(infofn))
+ return
+
try:
success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
except (OSError, IOError), err:
class YoutubeIE(InfoExtractor):
"""Information extractor for youtube.com."""
- _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
+ _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
except NameError:
video_description = u'No description available.'
if self._downloader.params.get('forcedescription', False) or self._downloader.params.get('writedescription', False):
- warnings.warn(u'You are using an old Python version, install Python 2.6+ or lxml. Falling back to old video description extractor.')
mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
if mobj is not None:
video_description = mobj.group(1).decode('utf-8')
html_parser = lxml.etree.HTMLParser(encoding='utf-8')
vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()'))
+ # TODO use another parser
# token
video_token = urllib.unquote_plus(video_info['token'][0])
# Decide which formats to download
req_format = self._downloader.params.get('format', None)
- if 'fmt_url_map' in video_info and len(video_info['fmt_url_map']) >= 1 and ',' in video_info['fmt_url_map'][0]:
- url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(','))
+ if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
+ self.report_rtmp_download()
+ video_url_list = [(None, video_info['conn'][0])]
+ elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
+ url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
+ url_data = [parse_qs(uds) for uds in url_data_strs]
+ url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
+ url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
+
format_limit = self._downloader.params.get('format_limit', None)
if format_limit is not None and format_limit in self._available_formats:
format_list = self._available_formats[self._available_formats.index(format_limit):]
self._downloader.trouble(u'ERROR: requested format not available')
return
video_url_list = [(req_format, url_map[req_format])] # Specific format
-
- elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
- self.report_rtmp_download()
- video_url_list = [(None, video_info['conn'][0])]
-
else:
- self._downloader.trouble(u'ERROR: no fmt_url_map or conn information found in video info')
+ self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
return
for format_param, video_real_url in video_url_list:
# Extension
video_extension = self._video_extensions.get(format_param, 'flv')
- # Find the video URL in fmt_url_map or conn paramters
try:
# Process video information
self._downloader.process_info({
# Extension
video_extension = self._video_extensions.get(format_param, 'mp4')
- # Find the video URL in fmt_url_map or conn paramters
try:
# Process video information
self._downloader.process_info({
self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
return
- json_url = url + ('&' if '?' in url else '?') + 'skin=json&version=2&no_wrap=1'
+ if '?' in url:
+ cchar = '&'
+ else:
+ cchar = '?'
+ json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
request = urllib2.Request(json_url)
self.report_extraction(mobj.group(1))
try:
return
try:
json_data = json.loads(json_code)
- data = json_data['Post'] if 'Post' in json_data else json_data
+ if 'Post' in json_data:
+ data = json_data['Post']
+ else:
+ data = json_data
upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
video_url = data['media']['url']
filesystem.add_option('--write-description',
action='store_true', dest='writedescription',
help='write video description to a .description file', default=False)
+ filesystem.add_option('--write-info-json',
+ action='store_true', dest='writeinfojson',
+ help='write video metadata to a .info.json file', default=False)
parser.add_option_group(filesystem)
postproc = optparse.OptionGroup(parser, 'Post-processing Options')
'nopart': opts.nopart,
'updatetime': opts.updatetime,
'writedescription': opts.writedescription,
+ 'writeinfojson': opts.writeinfojson,
})
fd.add_info_extractor(youtube_search_ie)
fd.add_info_extractor(youtube_pl_ie)
projects / youtube-dl / blobdiff