"""Information Extractor class.
Information extractors are the classes that, given a URL, extract
- information from the video (or videos) the URL refers to. This
- information includes the real video URL, the video title and simplified
- title, author and others. The information is stored in a dictionary
- which is then passed to the FileDownloader. The FileDownloader
- processes this information possibly downloading the video to the file
- system, among other possible outcomes. The dictionaries must include
- the following fields:
-
- id: Video identifier.
- url: Final video URL.
- uploader: Nickname of the video uploader.
- title: Literal title.
- ext: Video filename extension.
- format: Video format.
- player_url: SWF Player URL (may be None).
-
- The following fields are optional. Their primary purpose is to allow
- youtube-dl to serve as the backend for a video search function, such
- as the one in youtube2mp3. They are only used when their respective
- forced printing functions are called:
-
- thumbnail: Full URL to a video thumbnail image.
- description: One-line video description.
+ information about the video (or videos) the URL refers to. This
+ information includes the real video URL, the video title, author and
+ others. The information is stored in a dictionary which is then
+ passed to the FileDownloader. The FileDownloader processes this
+ information possibly downloading the video to the file system, among
+ other possible outcomes.
+
+ The dictionaries must include the following fields:
+
+ id: Video identifier.
+ url: Final video URL.
+ uploader: Nickname of the video uploader, unescaped.
+ upload_date: Video upload date (YYYYMMDD).
+ title: Video title, unescaped.
+ ext: Video filename extension.
+
+ The following fields are optional:
+
+ format: The video format, defaults to ext (used for --get-format)
+ thumbnail: Full URL to a video thumbnail image.
+ description: One-line video description.
+ player_url: SWF Player URL (used for rtmpdump).
+ subtitles: The .srt file contents.
+ urlhandle: [internal] The urlHandle to be used to download the file,
+ like returned by urllib2.urlopen
+
+ The fields should all be Unicode strings.
Subclasses of this one should re-define the _real_initialize() and
_real_extract() methods and define a _VALID_URL regexp.
Probably, they should also be added to the list of extractors.
+
+ _real_extract() must return a *list* of information dictionaries as
+ described above.
+
+ Finally, the _WORKING attribute should be set to False for broken IEs
+ in order to warn the users and skip the tests.
"""
_ready = False
_downloader = None
+ _WORKING = True
def __init__(self, downloader=None):
"""Constructor. Receives an optional downloader."""
"""Receives a URL and returns True if suitable for this IE."""
return re.match(self._VALID_URL, url) is not None
+ def working(self):
+ """Getter method for _WORKING."""
+ return self._WORKING
+
def initialize(self):
"""Initializes an instance (authentication, etc)."""
if not self._ready:
else:
raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
except (IOError, netrc.NetrcParseError), err:
- self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
+ self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % u(err))
return
# Set language
self.report_lang()
urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
+ self._downloader.to_stderr(u'WARNING: unable to set language: %s' % u(err))
return
# No authentication to be performed
self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
return
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
+ self._downloader.to_stderr(u'WARNING: unable to log in: %s' % u(err))
return
# Confirm age
self.report_age_confirmation()
age_results = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to confirm age: %s' % u(err))
return
def _real_extract(self, url):
try:
video_webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
return
# Attempt to extract SWF player URL
if 'token' in video_info:
break
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % u(err))
return
if 'token' not in video_info:
if 'reason' in video_info:
video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
# upload date
- upload_date = u'NA'
+ upload_date = None
mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
if mobj is not None:
upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
try:
srt_list = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
+ raise Trouble(u'WARNING: unable to download video subtitles: %s' % u(err))
srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
if not srt_lang_list:
try:
srt_xml = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
+ raise Trouble(u'WARNING: unable to download video subtitles: %s' % u(err))
if not srt_xml:
raise Trouble(u'WARNING: unable to download video subtitles')
video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
# Extension
video_extension = self._video_extensions.get(format_param, 'flv')
+ video_format = '{} - {}'.format(format_param.decode('utf-8') if format_param else video_extension.decode('utf-8'),
+ self._video_dimensions.get(format_param, '???'))
+
results.append({
'id': video_id.decode('utf-8'),
'url': video_real_url.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
'ext': video_extension.decode('utf-8'),
- 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
+ 'format': video_format,
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description,
'player_url': player_url,
self.report_disclaimer()
disclaimer = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % u(err))
return
# Confirm age
self.report_age_confirmation()
disclaimer = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to confirm age: %s' % u(err))
return
def _real_extract(self, url):
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % u(err))
return
# Extract URL, uploader and title from webpage
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
- 'upload_date': u'NA',
+ 'upload_date': None,
'title': video_title,
'ext': video_extension.decode('utf-8'),
- 'format': u'NA',
- 'player_url': None,
}]
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % u(err))
return
# Extract URL, uploader and title from webpage
return
video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
- video_uploader = u'NA'
+ video_uploader = None
mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
if mobj is None:
# lookin for official user
else:
video_uploader = mobj.group(1)
- video_upload_date = u'NA'
+ video_upload_date = None
mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
if mobj is not None:
video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
'upload_date': video_upload_date,
'title': video_title,
'ext': video_extension.decode('utf-8'),
- 'format': u'NA',
- 'player_url': None,
}]
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
# Extract URL, uploader, and title from webpage
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
if mobj is None:
return [{
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
- 'uploader': u'NA',
- 'upload_date': u'NA',
+ 'uploader': None,
+ 'upload_date': None,
'title': video_title,
'ext': video_extension.decode('utf-8'),
- 'format': u'NA',
- 'player_url': None,
}]
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
# Extract URL, uploader, and title from webpage
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader,
- 'upload_date': u'NA',
+ 'upload_date': None,
'title': video_title,
'ext': video_extension.decode('utf-8'),
- 'format': u'NA',
- 'player_url': None,
}]
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
# Extract uploader and title from webpage
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
# Extract media URL from playlist XML
'id': video_id.decode('utf-8'),
'url': video_url,
'uploader': video_uploader,
- 'upload_date': u'NA',
+ 'upload_date': None,
'title': video_title,
'ext': video_extension.decode('utf-8'),
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description,
- 'thumbnail': video_thumbnail,
- 'player_url': None,
}]
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
# Now we begin extracting as much information as we can from what we
else: video_description = ''
# Extract upload date
- video_upload_date = u'NA'
+ video_upload_date = None
mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
if mobj is not None:
video_upload_date = mobj.group(1)
'ext': video_extension,
'thumbnail': video_thumbnail,
'description': video_description,
- 'player_url': None,
}]
+class ArteTvIE(InfoExtractor):
+ """arte.tv information extractor."""
+
+ _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
+ _LIVE_URL = r'index-[0-9]+\.html$'
+
+ IE_NAME = u'arte.tv'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ def report_download_webpage(self, video_id):
+ """Report webpage download."""
+ self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
+
+ def fetch_webpage(self, url):
+ self._downloader.increment_downloads()
+ request = urllib2.Request(url)
+ try:
+ self.report_download_webpage(url)
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
+ return
+ except ValueError, err:
+ self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
+ return
+ return webpage
+
+ def grep_webpage(self, url, regex, regexFlags, matchTuples):
+ page = self.fetch_webpage(url)
+ mobj = re.search(regex, page, regexFlags)
+ info = {}
+
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
+ return
+
+ for (i, key, err) in matchTuples:
+ if mobj.group(i) is None:
+ self._downloader.trouble(err)
+ return
+ else:
+ info[key] = mobj.group(i)
+
+ return info
+
+ def extractLiveStream(self, url):
+ video_lang = url.split('/')[-4]
+ info = self.grep_webpage(
+ url,
+ r'src="(.*?/videothek_js.*?\.js)',
+ 0,
+ [
+ (1, 'url', u'ERROR: Invalid URL: %s' % url)
+ ]
+ )
+ http_host = url.split('/')[2]
+ next_url = 'http://%s%s' % (http_host, urllib.unquote(info.get('url')))
+ info = self.grep_webpage(
+ next_url,
+ r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
+ '(http://.*?\.swf).*?' +
+ '(rtmp://.*?)\'',
+ re.DOTALL,
+ [
+ (1, 'path', u'ERROR: could not extract video path: %s' % url),
+ (2, 'player', u'ERROR: could not extract video player: %s' % url),
+ (3, 'url', u'ERROR: could not extract video url: %s' % url)
+ ]
+ )
+ video_url = u'%s/%s' % (info.get('url'), info.get('path'))
+
+ def extractPlus7Stream(self, url):
+ video_lang = url.split('/')[-3]
+ info = self.grep_webpage(
+ url,
+ r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
+ 0,
+ [
+ (1, 'url', u'ERROR: Invalid URL: %s' % url)
+ ]
+ )
+ next_url = urllib.unquote(info.get('url'))
+ info = self.grep_webpage(
+ next_url,
+ r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
+ 0,
+ [
+ (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
+ ]
+ )
+ next_url = urllib.unquote(info.get('url'))
+
+ info = self.grep_webpage(
+ next_url,
+ r'<video id="(.*?)".*?>.*?' +
+ '<name>(.*?)</name>.*?' +
+ '<dateVideo>(.*?)</dateVideo>.*?' +
+ '<url quality="hd">(.*?)</url>',
+ re.DOTALL,
+ [
+ (1, 'id', u'ERROR: could not extract video id: %s' % url),
+ (2, 'title', u'ERROR: could not extract video title: %s' % url),
+ (3, 'date', u'ERROR: could not extract video date: %s' % url),
+ (4, 'url', u'ERROR: could not extract video url: %s' % url)
+ ]
+ )
+
+ return {
+ 'id': info.get('id'),
+ 'url': urllib.unquote(info.get('url')),
+ 'uploader': u'arte.tv',
+ 'upload_date': info.get('date'),
+ 'title': info.get('title'),
+ 'ext': u'mp4',
+ 'format': u'NA',
+ 'player_url': None,
+ }
+
+ def _real_extract(self, url):
+ video_id = url.split('/')[-1]
+ self.report_extraction(video_id)
+
+ if re.search(self._LIVE_URL, video_id) is not None:
+ self.extractLiveStream(url)
+ return
+ else:
+ info = self.extractPlus7Stream(url)
+
+ return [info]
+
+
class GenericIE(InfoExtractor):
"""Generic last-resort information extractor."""
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
except ValueError, err:
# since this is the last-resort InfoExtractor, if
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader,
- 'upload_date': u'NA',
+ 'upload_date': None,
'title': video_title,
'ext': video_extension.decode('utf-8'),
- 'format': u'NA',
- 'player_url': None,
}]
return
else:
try:
- n = long(prefix)
+ n = int(prefix)
if n <= 0:
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
return
try:
data = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download API page: %s' % u(err))
return
api_response = json.loads(data)['data']
return
else:
try:
- n = long(prefix)
+ n = int(prefix)
if n <= 0:
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
return
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
# Extract video identifiers
return
else:
try:
- n = long(prefix)
+ n = int(prefix)
if n <= 0:
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
return
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
# Extract video identifiers
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
# Extract video identifiers
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
# Extract video identifiers
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
# Extract video identifiers
mobj = re.search(r'data-users-id="([^"]+)"', page)
page_base = page_base % mobj.group(1)
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
self.report_download_webpage(file_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % u(err))
return
# Search for the real file URL
return [{
'id': file_id.decode('utf-8'),
'url': file_url.decode('utf-8'),
- 'uploader': u'NA',
- 'upload_date': u'NA',
+ 'uploader': None,
+ 'upload_date': None,
'title': file_title,
'ext': file_extension.decode('utf-8'),
- 'format': u'NA',
- 'player_url': None,
}]
class FacebookIE(InfoExtractor):
"""Information Extractor for Facebook"""
+ _WORKING = False
_VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
_LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
_NETRC_MACHINE = 'facebook'
else:
raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
except (IOError, netrc.NetrcParseError), err:
- self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
+ self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % u(err))
return
if useremail is None:
self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
return
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
+ self._downloader.to_stderr(u'WARNING: unable to log in: %s' % u(err))
return
def _real_extract(self, url):
page = urllib2.urlopen(request)
video_webpage = page.read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
return
# Start extracting information
video_thumbnail = video_info['thumbnail']
# upload date
- upload_date = u'NA'
+ upload_date = None
if 'upload_date' in video_info:
upload_time = video_info['upload_date']
timetuple = email.utils.parsedate_tz(upload_time)
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description.decode('utf-8'),
- 'player_url': None,
})
return results
info = {
'id': title,
'url': url,
+ 'uploader': None,
+ 'upload_date': None,
'title': title,
'ext': ext,
'urlhandle': urlh
}
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % u(err))
return
if info is None: # Regular URL
try:
json_code = urlh.read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % u(err))
return
try:
self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
self.report_extraction(video_id)
return [{
'id': video_id,
'url': video_url,
- 'uploader': u'NA',
- 'upload_date': u'NA',
+ 'uploader': None,
+ 'upload_date': None,
'title': video_title,
'ext': u'flv',
- 'format': u'NA',
- 'player_url': None,
}]
class ComedyCentralIE(InfoExtractor):
print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
-
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
if mobj is None:
htmlHandle = urllib2.urlopen(req)
html = htmlHandle.read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
if dlNewest:
url = htmlHandle.geturl()
urlHandle = urllib2.urlopen(playerUrl_raw)
playerUrl = urlHandle.geturl()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
+ self._downloader.trouble(u'ERROR: unable to find out player URL: ' + u(err))
return
uri = mMovieParams[0][1]
try:
indexXml = urllib2.urlopen(indexUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download episode index: ' + u(err))
return
results = []
try:
configXml = urllib2.urlopen(configReq).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
return
cdoc = xml.etree.ElementTree.fromstring(configXml)
continue
if self._downloader.params.get('listformats', None):
- self._print_formats([i[0] for i in turls])
- return
+ self._print_formats([i[0] for i in turls])
+ return
# For now, just pick the highest bitrate
format,video_url = turls[-1]
# Select format if we can find one
for f,v in turls:
- if f == req_format:
- format, video_url = f, v
- break
-
- # Patch to download from alternative CDN, which does not
- # break on current RTMPDump builds
-
+ if f == req_format:
+ format, video_url = f, v
+ break
+ # Patch to download from alternative CDN, which does not
+ # break on current RTMPDump builds
broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
-
+
if video_url.startswith(broken_cdn):
- video_url = video_url.replace(broken_cdn, better_cdn)
-
+ video_url = video_url.replace(broken_cdn, better_cdn)
effTitle = showId + u'-' + epTitle
info = {
m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download webpage: ' + u(err))
return
descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
try:
configJSON = urllib2.urlopen(configUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download configuration: ' + unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download configuration: ' + u(err))
return
# Technically, it's JavaScript, not JSON
try:
config = json.loads(configJSON)
except (ValueError,), err:
- self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + unicode(err))
+ self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + u(err))
return
playlist = config['playlist']
'upload_date': None,
'title': showName,
'ext': 'flv',
- 'format': 'flv',
'thumbnail': imgUrl,
'description': description,
'player_url': playerUrl,
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
return
m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
info = {
'id': video_id,
'internal_id': internal_video_id,
+ 'uploader': None,
+ 'upload_date': None,
}
self.report_extraction(video_id)
try:
metaXml = urllib2.urlopen(xmlUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % u(err))
return
mdoc = xml.etree.ElementTree.fromstring(metaXml)
info['url'] = videoNode.findall('./file')[0].text
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
info['ext'] = info['url'].rpartition('.')[2]
- info['format'] = info['ext']
except IndexError:
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
return
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
return
self.report_extraction(video_id)
'upload_date': None,
'title': video_title,
'ext': 'flv',
- 'format': 'flv',
'thumbnail': video_thumbnail,
'description': None,
- 'player_url': None,
}
return [info]
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
return
self.report_extraction('%s/%s' % (uploader, slug_title))
try:
upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
except Exception, e:
- self._downloader.to_stderr(compat_str(e))
+ self._downloader.to_stderr(u(e))
# for soundcloud, a request to a cross domain is required for cookies
request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
'upload_date': upload_date,
'title': title,
'ext': u'mp3',
- 'format': u'NA',
- 'player_url': None,
'description': description.decode('utf-8')
}]
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
return
self.report_extraction(url)
'uploader': None,
'upload_date': None,
'title': video_title,
- 'ext': extension,
- 'format': extension, # Extension is always(?) mp4, but seems to be flv
+ 'ext': extension, # Extension is always(?) mp4, but seems to be flv
'thumbnail': None,
'description': video_description,
- 'player_url': None,
}
return [info]
self.report_download_json(file_url)
jsonData = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % u(err))
return
# parse JSON
'id': file_id.decode('utf-8'),
'url': file_url.decode('utf-8'),
'uploader': uploader.decode('utf-8'),
- 'upload_date': u'NA',
+ 'upload_date': None,
'title': json_data['name'],
'ext': file_url.split('.')[-1].decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
video = mobj.group('video')
info = {
'id': course + '_' + video,
+ 'uploader': None,
+ 'upload_date': None,
}
self.report_extraction(info['id'])
try:
metaXml = urllib2.urlopen(xmlUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % u(err))
return
mdoc = xml.etree.ElementTree.fromstring(metaXml)
try:
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
return
info['ext'] = info['url'].rpartition('.')[2]
- info['format'] = info['ext']
return [info]
elif mobj.group('course'): # A course page
course = mobj.group('course')
info = {
'id': course,
'type': 'playlist',
+ 'uploader': None,
+ 'upload_date': None,
}
self.report_download_webpage(info['id'])
try:
coursepage = urllib2.urlopen(url).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download course info page: ' + u(err))
return
m = re.search('<h1>([^<]+)</h1>', coursepage)
info = {
'id': 'Stanford OpenClassroom',
'type': 'playlist',
+ 'uploader': None,
+ 'upload_date': None,
}
self.report_download_webpage(info['id'])
try:
rootpage = urllib2.urlopen(rootURL).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+ self._downloader.trouble(u'ERROR: unable to download course info page: ' + u(err))
return
info['title'] = info['id']
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
return
mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
try:
metadataXml = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % u(err))
return
mdoc = xml.etree.ElementTree.fromstring(metadataXml)
'id': video_id,
'url': video_url,
'uploader': performer,
+ 'upload_date': None,
'title': video_title,
'ext': ext,
'format': format,
self.report_download_webpage(video_id)
jsondata = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
self.report_extraction(video_id)
'id': '%s_part%02d' % (video_id, index),
'url': download_url,
'uploader': None,
+ 'upload_date': None,
'title': video_title,
'ext': ext,
- 'format': u'NA'
}
files_info.append(info)
return
video_thumbnail = result.group(1).decode('utf-8')
- info = {'id': video_id,
- 'url': video_url,
- 'uploader': None,
- 'upload_date': None,
- 'title': video_title,
- 'ext': 'flv',
- 'format': 'flv',
- 'thumbnail': video_thumbnail,
- 'description': None,
- 'player_url': None}
-
- return [info]
+ return [{
+ 'id': video_id,
+ 'url': video_url,
+ 'uploader': None,
+ 'upload_date': None,
+ 'title': video_title,
+ 'ext': 'flv',
+ 'thumbnail': video_thumbnail,
+ 'description': None,
+ }]
class GooglePlusIE(InfoExtractor):
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % u(err))
return
# Extract update date
- upload_date = u'NA'
+ upload_date = None
pattern = 'title="Timestamp">(.*?)</a>'
mobj = re.search(pattern, webpage)
if mobj:
self.report_date(upload_date)
# Extract uploader
- uploader = u'NA'
+ uploader = None
pattern = r'rel\="author".*?>(.*?)</a>'
mobj = re.search(pattern, webpage)
if mobj:
try:
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
return
self.report_extract_vid_page(video_page)
'upload_date': upload_date.decode('utf-8'),
'title': video_title.decode('utf-8'),
'ext': video_extension.decode('utf-8'),
- 'format': u'NA',
- 'player_url': None,
}]
projects / youtube-dl / blobdiff