X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=8b2442bacc21e86db7ffdc920116b51e79fd3820;hb=5e34d2ebbf9906bded4201d7bd8bb82e9353de9f;hp=d444e21d9c9acf32706b11c5be773ce6f6a60180;hpb=f17ce13a9260919c4d8cb652467023373c783bd2;p=youtube-dl

diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index d444e21d9..8b2442bac 100755
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -114,8 +114,8 @@ class InfoExtractor(object):
     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None):
         """ Returns the response handle """
         if note is None:
-            note = u'Downloading video webpage'
-        if note is not False:
+            self.report_download_webpage(video_id)
+        elif note is not False:
             self.to_screen(u'%s: %s' % (video_id, note))
         try:
             return compat_urllib_request.urlopen(url_or_request)
@@ -148,6 +148,18 @@ class InfoExtractor(object):
         """Print msg to screen, prefixing it with '[ie_name]'"""
         self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg))
 
+    def report_extraction(self, id_or_name):
+        """Report information extraction."""
+        self.to_screen(u'%s: Extracting information' % id_or_name)
+
+    def report_download_webpage(self, video_id):
+        """Report webpage download."""
+        self.to_screen(u'%s: Downloading webpage' % video_id)
+
+    def report_age_confirmation(self):
+        """Report attempt to confirm age."""
+        self.to_screen(u'Confirming age')
+
     #Methods for following #608
     #They set the correct value of the '_type' key
     def video_result(self, video_info):
@@ -246,10 +258,6 @@ class YoutubeIE(InfoExtractor):
         """Report attempt to log in."""
         self.to_screen(u'Logging in')
 
-    def report_age_confirmation(self):
-        """Report attempt to confirm age."""
-        self.to_screen(u'Confirming age')
-
     def report_video_webpage_download(self, video_id):
         """Report attempt to download video webpage."""
         self.to_screen(u'%s: Downloading video webpage' % video_id)
@@ -554,19 +562,18 @@ class YoutubeIE(InfoExtractor):
         mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
         if mobj is not None:
             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
-            format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
-            for expression in format_expressions:
-                try:
-                    upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
-                except:
-                    pass
+            upload_date = unified_strdate(upload_date)
 
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
             video_description = clean_html(video_description)
         else:
-            video_description = ''
+            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
+            if fd_mobj:
+                video_description = unescapeHTML(fd_mobj.group(1))
+            else:
+                video_description = u''
 
         # subtitles
         video_subtitles = None
@@ -618,8 +625,7 @@ class YoutubeIE(InfoExtractor):
                 format_list = available_formats
             existing_formats = [x for x in format_list if x in url_map]
             if len(existing_formats) == 0:
-                self._downloader.report_error(u'no known formats available for video')
-                return
+                raise ExtractorError(u'no known formats available for video')
             if self._downloader.params.get('listformats', None):
                 self._print_formats(existing_formats)
                 return
@@ -639,11 +645,9 @@ class YoutubeIE(InfoExtractor):
                         video_url_list = [(rf, url_map[rf])]
                         break
                 if video_url_list is None:
-                    self._downloader.report_error(u'requested format not available')
-                    return
+                    raise ExtractorError(u'requested format not available')
         else:
-            self._downloader.report_error(u'no conn or url_encoded_fmt_stream_map information found in video info')
-            return
+            raise ExtractorError(u'no conn or url_encoded_fmt_stream_map information found in video info')
 
         results = []
         for format_param, video_real_url in video_url_list:
@@ -679,25 +683,10 @@ class MetacafeIE(InfoExtractor):
     _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
     IE_NAME = u'metacafe'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_disclaimer(self):
         """Report disclaimer retrieval."""
         self.to_screen(u'Retrieving disclaimer')
 
-    def report_age_confirmation(self):
-        """Report attempt to confirm age."""
-        self.to_screen(u'Confirming age')
-
-    def report_download_webpage(self, video_id):
-        """Report webpage download."""
-        self.to_screen(u'%s: Downloading webpage' % video_id)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_initialize(self):
         # Retrieve disclaimer
         request = compat_urllib_request.Request(self._DISCLAIMER)
@@ -790,20 +779,11 @@ class MetacafeIE(InfoExtractor):
             'ext':      video_extension.decode('utf-8'),
         }]
 
-
 class DailymotionIE(InfoExtractor):
     """Information Extractor for Dailymotion"""
 
     _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
     IE_NAME = u'dailymotion'
-    _WORKING = False
-
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
 
     def _real_extract(self, url):
         # Extract id and simplified title from URL
@@ -886,17 +866,6 @@ class PhotobucketIE(InfoExtractor):
     _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
     IE_NAME = u'photobucket'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_download_webpage(self, video_id):
-        """Report webpage download."""
-        self.to_screen(u'%s: Downloading webpage' % video_id)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url):
         # Extract id from URL
         mobj = re.match(self._VALID_URL, url)
@@ -955,17 +924,6 @@ class YahooIE(InfoExtractor):
     _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
     IE_NAME = u'video.yahoo'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_download_webpage(self, video_id):
-        """Report webpage download."""
-        self.to_screen(u'%s: Downloading webpage' % video_id)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url, new_video=True):
         # Extract ID from URL
         mobj = re.match(self._VALID_URL, url)
@@ -1095,17 +1053,6 @@ class VimeoIE(InfoExtractor):
     _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|player)\.)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)'
     IE_NAME = u'vimeo'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_download_webpage(self, video_id):
-        """Report webpage download."""
-        self.to_screen(u'%s: Downloading webpage' % video_id)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url, new_video=True):
         # Extract ID from URL
         mobj = re.match(self._VALID_URL, url)
@@ -1121,13 +1068,7 @@ class VimeoIE(InfoExtractor):
 
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, std_headers)
-        try:
-            self.report_download_webpage(video_id)
-            webpage_bytes = compat_urllib_request.urlopen(request).read()
-            webpage = webpage_bytes.decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err))
-            return
+        webpage = self._download_webpage(request, video_id)
 
         # Now we begin extracting as much information as we can from what we
         # retrieved. First we extract the information common to all extractors,
@@ -1139,7 +1080,10 @@ class VimeoIE(InfoExtractor):
             config = webpage.split(' = {config:')[1].split(',assets:')[0]
             config = json.loads(config)
         except:
-            self._downloader.report_error(u'unable to extract info section')
+            if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
+                self._downloader.report_error(u'The author has restricted the access to this video, try with the "--referer" option')
+            else:
+                self._downloader.report_error(u'unable to extract info section')
             return
 
         # Extract title
@@ -1216,17 +1160,6 @@ class ArteTvIE(InfoExtractor):
 
     IE_NAME = u'arte.tv'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_download_webpage(self, video_id):
-        """Report webpage download."""
-        self.to_screen(u'%s: Downloading webpage' % video_id)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def fetch_webpage(self, url):
         request = compat_urllib_request.Request(url)
         try:
@@ -1251,7 +1184,7 @@ class ArteTvIE(InfoExtractor):
 
         for (i, key, err) in matchTuples:
             if mobj.group(i) is None:
-                self._downloader.trouble(err)
+                self._downloader.report_error(err)
                 return
             else:
                 info[key] = mobj.group(i)
@@ -1265,7 +1198,7 @@ class ArteTvIE(InfoExtractor):
             r'src="(.*?/videothek_js.*?\.js)',
             0,
             [
-                (1, 'url', u'ERROR: Invalid URL: %s' % url)
+                (1, 'url', u'Invalid URL: %s' % url)
             ]
         )
         http_host = url.split('/')[2]
@@ -1277,9 +1210,9 @@ class ArteTvIE(InfoExtractor):
                 '(rtmp://.*?)\'',
             re.DOTALL,
             [
-                (1, 'path',   u'ERROR: could not extract video path: %s' % url),
-                (2, 'player', u'ERROR: could not extract video player: %s' % url),
-                (3, 'url',    u'ERROR: could not extract video url: %s' % url)
+                (1, 'path',   u'could not extract video path: %s' % url),
+                (2, 'player', u'could not extract video player: %s' % url),
+                (3, 'url',    u'could not extract video url: %s' % url)
             ]
         )
         video_url = u'%s/%s' % (info.get('url'), info.get('path'))
@@ -1291,7 +1224,7 @@ class ArteTvIE(InfoExtractor):
             r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
             0,
             [
-                (1, 'url', u'ERROR: Invalid URL: %s' % url)
+                (1, 'url', u'Invalid URL: %s' % url)
             ]
         )
         next_url = compat_urllib_parse.unquote(info.get('url'))
@@ -1300,7 +1233,7 @@ class ArteTvIE(InfoExtractor):
             r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
             0,
             [
-                (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
+                (1, 'url', u'Could not find <video> tag: %s' % url)
             ]
         )
         next_url = compat_urllib_parse.unquote(info.get('url'))
@@ -1313,10 +1246,10 @@ class ArteTvIE(InfoExtractor):
                 '<url quality="hd">(.*?)</url>',
             re.DOTALL,
             [
-                (1, 'id',    u'ERROR: could not extract video id: %s' % url),
-                (2, 'title', u'ERROR: could not extract video title: %s' % url),
-                (3, 'date',  u'ERROR: could not extract video date: %s' % url),
-                (4, 'url',   u'ERROR: could not extract video url: %s' % url)
+                (1, 'id',    u'could not extract video id: %s' % url),
+                (2, 'title', u'could not extract video title: %s' % url),
+                (3, 'date',  u'could not extract video date: %s' % url),
+                (4, 'url',   u'could not extract video url: %s' % url)
             ]
         )
 
@@ -1350,18 +1283,11 @@ class GenericIE(InfoExtractor):
     _VALID_URL = r'.*'
     IE_NAME = u'generic'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_download_webpage(self, video_id):
         """Report webpage download."""
         if not self._downloader.params.get('test', False):
             self._downloader.report_warning(u'Falling back on generic information extractor.')
-        self.to_screen(u'%s: Downloading webpage' % video_id)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
+        super(GenericIE, self).report_download_webpage(video_id)
 
     def report_following_redirect(self, new_url):
         """Report information extraction."""
@@ -1496,9 +1422,6 @@ class YoutubeSearchIE(InfoExtractor):
     _max_youtube_results = 1000
     IE_NAME = u'youtube:search'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_download_page(self, query, pagenum):
         """Report attempt to download search page with given number."""
         query = query.decode(preferredencoding())
@@ -1549,7 +1472,7 @@ class YoutubeSearchIE(InfoExtractor):
             api_response = json.loads(data)['data']
 
             if not 'items' in api_response:
-                self._downloader.trouble(u'[youtube] No video results')
+                self._downloader.report_error(u'[youtube] No video results')
                 return
 
             new_ids = list(video['id'] for video in api_response['items'])
@@ -1573,9 +1496,6 @@ class GoogleSearchIE(InfoExtractor):
     _max_google_results = 1000
     IE_NAME = u'video.google:search'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_download_page(self, query, pagenum):
         """Report attempt to download playlist page with given number."""
         query = query.decode(preferredencoding())
@@ -1657,9 +1577,6 @@ class YahooSearchIE(InfoExtractor):
     _max_yahoo_results = 1000
     IE_NAME = u'video.yahoo:search'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_download_page(self, query, pagenum):
         """Report attempt to download playlist page with given number."""
         query = query.decode(preferredencoding())
@@ -1753,18 +1670,11 @@ class YoutubePlaylistIE(InfoExtractor):
     _MAX_RESULTS = 50
     IE_NAME = u'youtube:playlist'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     @classmethod
     def suitable(cls, url):
         """Receives a URL and returns True if suitable for this IE."""
         return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
 
-    def report_download_page(self, playlist_id, pagenum):
-        """Report attempt to download playlist page with given number."""
-        self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
-
     def _real_extract(self, url):
         # Extract playlist id
         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
@@ -1778,14 +1688,8 @@ class YoutubePlaylistIE(InfoExtractor):
         videos = []
 
         while True:
-            self.report_download_page(playlist_id, page_num)
-
             url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, self._MAX_RESULTS * (page_num - 1) + 1)
-            try:
-                page = compat_urllib_request.urlopen(url).read().decode('utf8')
-            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
-                return
+            page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
 
             try:
                 response = json.loads(page)
@@ -1796,12 +1700,11 @@ class YoutubePlaylistIE(InfoExtractor):
             if 'feed' not in response:
                 self._downloader.report_error(u'Got a malformed response from YouTube API')
                 return
+            playlist_title = response['feed']['title']['$t']
             if 'entry' not in response['feed']:
                 # Number of videos is a multiple of self._MAX_RESULTS
                 break
 
-            playlist_title = response['feed']['title']['$t']
-
             videos += [ (entry['yt$position']['$t'], entry['content']['src'])
                         for entry in response['feed']['entry']
                         if 'content' in entry ]
@@ -1825,10 +1728,6 @@ class YoutubeChannelIE(InfoExtractor):
     _MORE_PAGES_URL = 'http://www.youtube.com/channel_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
     IE_NAME = u'youtube:channel'
 
-    def report_download_page(self, channel_id, pagenum):
-        """Report attempt to download channel page with given number."""
-        self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
-
     def extract_videos_from_page(self, page):
         ids_in_page = []
         for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
@@ -1848,14 +1747,9 @@ class YoutubeChannelIE(InfoExtractor):
         video_ids = []
         pagenum = 1
 
-        self.report_download_page(channel_id, pagenum)
         url = self._TEMPLATE_URL % (channel_id, pagenum)
-        request = compat_urllib_request.Request(url)
-        try:
-            page = compat_urllib_request.urlopen(request).read().decode('utf8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
-            return
+        page = self._download_webpage(url, channel_id,
+                                      u'Downloading page #%s' % pagenum)
 
         # Extract video identifiers
         ids_in_page = self.extract_videos_from_page(page)
@@ -1866,14 +1760,9 @@ class YoutubeChannelIE(InfoExtractor):
             while True:
                 pagenum = pagenum + 1
 
-                self.report_download_page(channel_id, pagenum)
                 url = self._MORE_PAGES_URL % (pagenum, channel_id)
-                request = compat_urllib_request.Request(url)
-                try:
-                    page = compat_urllib_request.urlopen(request).read().decode('utf8')
-                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                    self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
-                    return
+                page = self._download_webpage(url, channel_id,
+                                              u'Downloading page #%s' % pagenum)
 
                 page = json.loads(page)
 
@@ -1900,14 +1789,6 @@ class YoutubeUserIE(InfoExtractor):
     _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
     IE_NAME = u'youtube:user'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_download_page(self, username, start_index):
-        """Report attempt to download user page."""
-        self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
-                (username, start_index, start_index + self._GDATA_PAGE_SIZE))
-
     def _real_extract(self, url):
         # Extract username
         mobj = re.match(self._VALID_URL, url)
@@ -1927,15 +1808,10 @@ class YoutubeUserIE(InfoExtractor):
 
         while True:
             start_index = pagenum * self._GDATA_PAGE_SIZE + 1
-            self.report_download_page(username, start_index)
 
-            request = compat_urllib_request.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
-
-            try:
-                page = compat_urllib_request.urlopen(request).read().decode('utf-8')
-            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
-                return
+            gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
+            page = self._download_webpage(gdata_url, username,
+                                          u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
 
             # Extract video identifiers
             ids_in_page = []
@@ -1969,14 +1845,6 @@ class BlipTVUserIE(InfoExtractor):
     _PAGE_SIZE = 12
     IE_NAME = u'blip.tv:user'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_download_page(self, username, pagenum):
-        """Report attempt to download user page."""
-        self.to_screen(u'user %s: Downloading video ids from page %d' %
-                (username, pagenum))
-
     def _real_extract(self, url):
         # Extract username
         mobj = re.match(self._VALID_URL, url)
@@ -1988,15 +1856,9 @@ class BlipTVUserIE(InfoExtractor):
 
         page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
 
-        request = compat_urllib_request.Request(url)
-
-        try:
-            page = compat_urllib_request.urlopen(request).read().decode('utf-8')
-            mobj = re.search(r'data-users-id="([^"]+)"', page)
-            page_base = page_base % mobj.group(1)
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
-            return
+        page = self._download_webpage(url, username, u'Downloading user page')
+        mobj = re.search(r'data-users-id="([^"]+)"', page)
+        page_base = page_base % mobj.group(1)
 
 
         # Download video ids using BlipTV Ajax calls. Result size per
@@ -2008,14 +1870,9 @@ class BlipTVUserIE(InfoExtractor):
         pagenum = 1
 
         while True:
-            self.report_download_page(username, pagenum)
             url = page_base + "&page=" + str(pagenum)
-            request = compat_urllib_request.Request( url )
-            try:
-                page = compat_urllib_request.urlopen(request).read().decode('utf-8')
-            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                self._downloader.report_error(u'unable to download webpage: %s' % str(err))
-                return
+            page = self._download_webpage(url, username,
+                                          u'Downloading video ids from page %d' % pagenum)
 
             # Extract video identifiers
             ids_in_page = []
@@ -2047,14 +1904,6 @@ class DepositFilesIE(InfoExtractor):
 
     _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
 
-    def report_download_webpage(self, file_id):
-        """Report webpage download."""
-        self.to_screen(u'%s: Downloading webpage' % file_id)
-
-    def report_extraction(self, file_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % file_id)
-
     def _real_extract(self, url):
         file_id = url.split('/')[-1]
         # Rebuild url in english locale
@@ -2208,10 +2057,6 @@ class BlipTVIE(InfoExtractor):
     _URL_EXT = r'^.*\.([a-z0-9]+)$'
     IE_NAME = u'blip.tv'
 
-    def report_extraction(self, file_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % file_id)
-
     def report_direct_download(self, title):
         """Report information extraction."""
         self.to_screen(u'%s: Direct download detected' % title)
@@ -2309,13 +2154,6 @@ class MyVideoIE(InfoExtractor):
     _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
     IE_NAME = u'myvideo'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self,url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -2393,15 +2231,6 @@ class ComedyCentralIE(InfoExtractor):
         """Receives a URL and returns True if suitable for this IE."""
         return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
 
-    def report_extraction(self, episode_id):
-        self.to_screen(u'%s: Extracting information' % episode_id)
-
-    def report_config_download(self, episode_id, media_id):
-        self.to_screen(u'%s: Downloading configuration for %s' % (episode_id, media_id))
-
-    def report_index_download(self, episode_id):
-        self.to_screen(u'%s: Downloading show index' % episode_id)
-
     def _print_formats(self, formats):
         print('Available formats:')
         for x in formats:
@@ -2435,15 +2264,8 @@ class ComedyCentralIE(InfoExtractor):
             else:
                 epTitle = mobj.group('episode')
 
-        req = compat_urllib_request.Request(url)
         self.report_extraction(epTitle)
-        try:
-            htmlHandle = compat_urllib_request.urlopen(req)
-            html = htmlHandle.read()
-            webpage = html.decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
-            return
+        webpage = self._download_webpage(url, epTitle)
         if dlNewest:
             url = htmlHandle.geturl()
             mobj = re.match(self._VALID_URL, url, re.VERBOSE)
@@ -2471,12 +2293,9 @@ class ComedyCentralIE(InfoExtractor):
 
         uri = mMovieParams[0][1]
         indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})
-        self.report_index_download(epTitle)
-        try:
-            indexXml = compat_urllib_request.urlopen(indexUrl).read()
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download episode index: ' + compat_str(err))
-            return
+        indexXml = self._download_webpage(indexUrl, epTitle,
+                                          u'Downloading show index',
+                                          u'unable to download episode index')
 
         results = []
 
@@ -2487,17 +2306,12 @@ class ComedyCentralIE(InfoExtractor):
             shortMediaId = mediaId.split(':')[-1]
             showId = mediaId.split(':')[-2].replace('.com', '')
             officialTitle = itemEl.findall('./title')[0].text
-            officialDate = itemEl.findall('./pubDate')[0].text
+            officialDate = unified_strdate(itemEl.findall('./pubDate')[0].text)
 
             configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
                         compat_urllib_parse.urlencode({'uri': mediaId}))
-            configReq = compat_urllib_request.Request(configUrl)
-            self.report_config_download(epTitle, shortMediaId)
-            try:
-                configXml = compat_urllib_request.urlopen(configReq).read()
-            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
-                return
+            configXml = self._download_webpage(configUrl, epTitle,
+                                               u'Downloading configuration for %s' % shortMediaId)
 
             cdoc = xml.etree.ElementTree.fromstring(configXml)
             turls = []
@@ -2554,12 +2368,6 @@ class EscapistIE(InfoExtractor):
     _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
     IE_NAME = u'escapist'
 
-    def report_extraction(self, showName):
-        self.to_screen(u'%s: Extracting information' % showName)
-
-    def report_config_download(self, showName):
-        self.to_screen(u'%s: Downloading configuration' % showName)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -2569,14 +2377,7 @@ class EscapistIE(InfoExtractor):
         videoId = mobj.group('episode')
 
         self.report_extraction(showName)
-        try:
-            webPage = compat_urllib_request.urlopen(url)
-            webPageBytes = webPage.read()
-            m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
-            webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download webpage: ' + compat_str(err))
-            return
+        webPage = self._download_webpage(url, showName)
 
         descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
         description = unescapeHTML(descMatch.group(1))
@@ -2587,14 +2388,9 @@ class EscapistIE(InfoExtractor):
         configUrlMatch = re.search('config=(.*)$', playerUrl)
         configUrl = compat_urllib_parse.unquote(configUrlMatch.group(1))
 
-        self.report_config_download(showName)
-        try:
-            configJSON = compat_urllib_request.urlopen(configUrl)
-            m = re.match(r'text/html; charset="?([^"]+)"?', configJSON.headers['Content-Type'])
-            configJSON = configJSON.read().decode(m.group(1) if m else 'utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download configuration: ' + compat_str(err))
-            return
+        configJSON = self._download_webpage(configUrl, showName,
+                                            u'Downloading configuration',
+                                            u'unable to download configuration')
 
         # Technically, it's JavaScript, not JSON
         configJSON = configJSON.replace("'", '"')
@@ -2633,10 +2429,6 @@ class CollegeHumorIE(InfoExtractor):
         """Report information extraction."""
         self.to_screen(u'%s: Downloading XML manifest' % video_id)
 
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -2700,10 +2492,6 @@ class XVideosIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
     IE_NAME = u'xvideos'
 
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -2765,17 +2553,10 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
     IE_NAME = u'soundcloud'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_resolve(self, video_id):
         """Report information extraction."""
         self.to_screen(u'%s: Resolving id' % video_id)
 
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Retrieving stream' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -2787,40 +2568,32 @@ class SoundcloudIE(InfoExtractor):
         # extract simple title (uploader + slug of song title)
         slug_title =  mobj.group(2)
         simple_title = uploader + u'-' + slug_title
+        full_title = '%s/%s' % (uploader, slug_title)
 
-        self.report_resolve('%s/%s' % (uploader, slug_title))
+        self.report_resolve(full_title)
 
         url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
         resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
-        request = compat_urllib_request.Request(resolv_url)
-        try:
-            info_json_bytes = compat_urllib_request.urlopen(request).read()
-            info_json = info_json_bytes.decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download video webpage: %s' % compat_str(err))
-            return
+        info_json = self._download_webpage(resolv_url, full_title, u'Downloading info JSON')
 
         info = json.loads(info_json)
         video_id = info['id']
-        self.report_extraction('%s/%s' % (uploader, slug_title))
+        self.report_extraction(full_title)
 
         streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
-        request = compat_urllib_request.Request(streams_url)
-        try:
-            stream_json_bytes = compat_urllib_request.urlopen(request).read()
-            stream_json = stream_json_bytes.decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download stream definitions: %s' % compat_str(err))
-            return
+        stream_json = self._download_webpage(streams_url, full_title,
+                                             u'Downloading stream definitions',
+                                             u'unable to download stream definitions')
 
         streams = json.loads(stream_json)
         mediaURL = streams['http_mp3_128_url']
+        upload_date = unified_strdate(info['created_at'])
 
         return [{
             'id':       info['id'],
             'url':      mediaURL,
             'uploader': info['user']['username'],
-            'upload_date':  info['created_at'],
+            'upload_date': upload_date,
             'title':    info['title'],
             'ext':      u'mp3',
             'description': info['description'],
@@ -2836,23 +2609,16 @@ class SoundcloudSetIE(InfoExtractor):
      """
 
     _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)'
-    IE_NAME = u'soundcloud'
-
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
+    IE_NAME = u'soundcloud:set'
 
     def report_resolve(self, video_id):
         """Report information extraction."""
         self.to_screen(u'%s: Resolving id' % video_id)
 
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Retrieving stream' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
-            self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+            self._downloader.report_error(u'invalid URL: %s' % url)
             return
 
         # extract uploader (which is in the url)
@@ -2860,39 +2626,29 @@ class SoundcloudSetIE(InfoExtractor):
         # extract simple title (uploader + slug of song title)
         slug_title =  mobj.group(2)
         simple_title = uploader + u'-' + slug_title
+        full_title = '%s/sets/%s' % (uploader, slug_title)
 
-        self.report_resolve('%s/sets/%s' % (uploader, slug_title))
+        self.report_resolve(full_title)
 
         url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
         resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
-        request = compat_urllib_request.Request(resolv_url)
-        try:
-            info_json_bytes = compat_urllib_request.urlopen(request).read()
-            info_json = info_json_bytes.decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
-            return
+        info_json = self._download_webpage(resolv_url, full_title)
 
         videos = []
         info = json.loads(info_json)
         if 'errors' in info:
             for err in info['errors']:
-                self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err['error_message']))
+                self._downloader.report_error(u'unable to download video webpage: %s' % compat_str(err['error_message']))
             return
 
+        self.report_extraction(full_title)
         for track in info['tracks']:
             video_id = track['id']
-            self.report_extraction('%s/sets/%s' % (uploader, slug_title))
 
             streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
-            request = compat_urllib_request.Request(streams_url)
-            try:
-                stream_json_bytes = compat_urllib_request.urlopen(request).read()
-                stream_json = stream_json_bytes.decode('utf-8')
-            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                self._downloader.trouble(u'ERROR: unable to download stream definitions: %s' % compat_str(err))
-                return
+            stream_json = self._download_webpage(streams_url, video_id, u'Downloading track info JSON')
 
+            self.report_extraction(video_id)
             streams = json.loads(stream_json)
             mediaURL = streams['http_mp3_128_url']
 
@@ -2900,7 +2656,7 @@ class SoundcloudSetIE(InfoExtractor):
                 'id':       video_id,
                 'url':      mediaURL,
                 'uploader': track['user']['username'],
-                'upload_date':  track['created_at'],
+                'upload_date':  unified_strdate(track['created_at']),
                 'title':    track['title'],
                 'ext':      u'mp3',
                 'description': track['description'],
@@ -2912,10 +2668,6 @@ class InfoQIE(InfoExtractor):
     """Information extractor for infoq.com"""
     _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
 
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -2926,7 +2678,7 @@ class InfoQIE(InfoExtractor):
         self.report_extraction(url)
 
         # Extract video URL
-        mobj = re.search(r"jsclassref='([^']*)'", webpage)
+        mobj = re.search(r"jsclassref ?= ?'([^']*)'", webpage)
         if mobj is None:
             self._downloader.report_error(u'unable to extract video url')
             return
@@ -2969,17 +2721,10 @@ class MixcloudIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
     IE_NAME = u'mixcloud'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_download_json(self, file_id):
         """Report JSON download."""
         self.to_screen(u'Downloading json')
 
-    def report_extraction(self, file_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % file_id)
-
     def get_urls(self, jsonData, fmt, bitrate='best'):
         """Get urls from 'audio_formats' section in json"""
         file_url = None
@@ -3083,14 +2828,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
     IE_NAME = u'stanfordoc'
 
-    def report_download_webpage(self, objid):
-        """Report information extraction."""
-        self.to_screen(u'%s: Downloading webpage' % objid)
-
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -3195,10 +2932,6 @@ class MTVIE(InfoExtractor):
     _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
     IE_NAME = u'mtv'
 
-    def report_extraction(self, video_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -3254,7 +2987,7 @@ class MTVIE(InfoExtractor):
             format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
             video_url = rendition.find('./src').text
         except KeyError:
-            self._downloader.trouble('Invalid rendition field.')
+            self._downloader.report_error('Invalid rendition field.')
             return
 
         info = {
@@ -3273,14 +3006,6 @@ class MTVIE(InfoExtractor):
 class YoukuIE(InfoExtractor):
     _VALID_URL =  r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
 
-    def report_download_webpage(self, file_id):
-        """Report webpage download."""
-        self.to_screen(u'%s: Downloading webpage' % file_id)
-
-    def report_extraction(self, file_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % file_id)
-
     def _gen_sid(self):
         nowTime = int(time.time() * 1000)
         random1 = random.randint(1000,1998)
@@ -3318,18 +3043,11 @@ class YoukuIE(InfoExtractor):
 
         info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
 
-        request = compat_urllib_request.Request(info_url, None, std_headers)
-        try:
-            self.report_download_webpage(video_id)
-            jsondata = compat_urllib_request.urlopen(request).read()
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err))
-            return
+        jsondata = self._download_webpage(info_url, video_id)
 
         self.report_extraction(video_id)
         try:
-            jsonstr = jsondata.decode('utf-8')
-            config = json.loads(jsonstr)
+            config = json.loads(jsondata)
 
             video_title =  config['data'][0]['title']
             seed = config['data'][0]['seed']
@@ -3390,14 +3108,6 @@ class XNXXIE(InfoExtractor):
     VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
     VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'
 
-    def report_webpage(self, video_id):
-        """Report information extraction"""
-        self.to_screen(u'%s: Downloading webpage' % video_id)
-
-    def report_extraction(self, video_id):
-        """Report information extraction"""
-        self.to_screen(u'%s: Extracting information' % video_id)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
@@ -3405,15 +3115,8 @@ class XNXXIE(InfoExtractor):
             return
         video_id = mobj.group(1)
 
-        self.report_webpage(video_id)
-
         # Get webpage content
-        try:
-            webpage_bytes = compat_urllib_request.urlopen(url).read()
-            webpage = webpage_bytes.decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download video webpage: %s' % err)
-            return
+        webpage = self._download_webpage(url, video_id)
 
         result = re.search(self.VIDEO_URL_RE, webpage)
         if result is None:
@@ -3451,9 +3154,6 @@ class GooglePlusIE(InfoExtractor):
     _VALID_URL = r'(?:https://)?plus\.google\.com/(?:[^/]+/)*?posts/(\w+)'
     IE_NAME = u'plus.google'
 
-    def __init__(self, downloader=None):
-        InfoExtractor.__init__(self, downloader)
-
     def report_extract_entry(self, url):
         """Report downloading extry"""
         self.to_screen(u'Downloading entry: %s' % url)
@@ -3488,12 +3188,7 @@ class GooglePlusIE(InfoExtractor):
 
         # Step 1, Retrieve post webpage to extract further information
         self.report_extract_entry(post_url)
-        request = compat_urllib_request.Request(post_url)
-        try:
-            webpage = compat_urllib_request.urlopen(request).read().decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'Unable to retrieve entry webpage: %s' % compat_str(err))
-            return
+        webpage = self._download_webpage(post_url, video_id, u'Downloading entry webpage')
 
         # Extract update date
         upload_date = None
@@ -3530,12 +3225,7 @@ class GooglePlusIE(InfoExtractor):
             self._downloader.report_error(u'unable to extract video page URL')
 
         video_page = mobj.group(1)
-        request = compat_urllib_request.Request(video_page)
-        try:
-            webpage = compat_urllib_request.urlopen(request).read().decode('utf-8')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'Unable to retrieve video webpage: %s' % compat_str(err))
-            return
+        webpage = self._download_webpage(video_page, video_id, u'Downloading video page')
         self.report_extract_vid_page(video_page)
 
 
@@ -3616,24 +3306,16 @@ class JustinTVIE(InfoExtractor):
     _JUSTIN_PAGE_LIMIT = 100
     IE_NAME = u'justin.tv'
 
-    def report_extraction(self, file_id):
-        """Report information extraction."""
-        self.to_screen(u'%s: Extracting information' % file_id)
-
     def report_download_page(self, channel, offset):
         """Report attempt to download a single page of videos."""
         self.to_screen(u'%s: Downloading video information from %d to %d' %
                 (channel, offset, offset + self._JUSTIN_PAGE_LIMIT))
 
     # Return count of items, list of *valid* items
-    def _parse_page(self, url):
-        try:
-            urlh = compat_urllib_request.urlopen(url)
-            webpage_bytes = urlh.read()
-            webpage = webpage_bytes.decode('utf-8', 'ignore')
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_error(u'unable to download video info JSON: %s' % compat_str(err))
-            return
+    def _parse_page(self, url, video_id):
+        webpage = self._download_webpage(url, video_id,
+                                         u'Downloading video info JSON',
+                                         u'unable to download video info JSON')
 
         response = json.loads(webpage)
         if type(response) != list:
@@ -3685,7 +3367,7 @@ class JustinTVIE(InfoExtractor):
             if paged:
                 self.report_download_page(video_id, offset)
             page_url = api + ('?offset=%d&limit=%d' % (offset, limit))
-            page_count, page_info = self._parse_page(page_url)
+            page_count, page_info = self._parse_page(page_url, video_id)
             info.extend(page_info)
             if not paged or page_count != limit:
                 break
@@ -3713,7 +3395,7 @@ class FunnyOrDieIE(InfoExtractor):
         if not m:
             m = re.search(r'<title>(?P<title>[^<]+?)</title>', webpage)
             if not m:
-                self._downloader.trouble(u'Cannot find video title')
+                self._downloader.report_error(u'Cannot find video title')
         title = clean_html(m.group('title'))
 
         m = re.search(r'<meta property="og:description" content="(?P<desc>.*?)"', webpage)
@@ -3732,7 +3414,8 @@ class FunnyOrDieIE(InfoExtractor):
         return [info]
 
 class SteamIE(InfoExtractor):
-    _VALID_URL = r"""http://store.steampowered.com/
+    _VALID_URL = r"""http://store\.steampowered\.com/
+                (agecheck/)?
                 (?P<urltype>video|app)/ #If the page is only for videos or for a game
                 (?P<gameID>\d+)/?
                 (?P<videoID>\d*)(?P<extra>\??) #For urltype == video we sometimes get the videoID
@@ -3745,10 +3428,13 @@ class SteamIE(InfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url, re.VERBOSE)
-        urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},"
         gameID = m.group('gameID')
-        videourl = 'http://store.steampowered.com/video/%s/' % gameID
+        videourl = 'http://store.steampowered.com/agecheck/video/%s/?snr=1_agecheck_agecheck__age-gate&ageDay=1&ageMonth=January&ageYear=1970' % gameID
+        self.report_age_confirmation()
         webpage = self._download_webpage(videourl, gameID)
+        game_title = re.search(r'<h2 class="pageheader">(?P<game_title>.*?)</h2>', webpage).group('game_title')
+        
+        urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},"
         mweb = re.finditer(urlRE, webpage)
         namesRE = r'<span class="title">(?P<videoName>.+?)</span>'
         titles = re.finditer(namesRE, webpage)
@@ -3770,7 +3456,7 @@ class SteamIE(InfoExtractor):
                 'thumbnail': video_thumb
                   }
             videos.append(info)
-        return videos
+        return [self.playlist_result(videos, gameID, game_title)]
 
 class UstreamIE(InfoExtractor):
     _VALID_URL = r'https?://www\.ustream\.tv/recorded/(?P<videoID>\d+)'
@@ -3799,16 +3485,15 @@ class WorldStarHipHopIE(InfoExtractor):
     IE_NAME = u'WorldStarHipHop'
 
     def _real_extract(self, url):
-        _src_url = r"""(http://hw-videos.*(?:mp4|flv))"""
-
-        webpage_src = compat_urllib_request.urlopen(url).read()
-        webpage_src = webpage_src.decode('utf-8')
-
-        mobj = re.search(_src_url, webpage_src)
+        _src_url = r"""(http://(hw-videos|hw-post1).*(?:mp4|flv))"""
 
         m = re.match(self._VALID_URL, url)
         video_id = m.group('id')
 
+        webpage_src = self._download_webpage(url, video_id) 
+
+        mobj = re.search(_src_url, webpage_src)
+
         if mobj is not None:
             video_url = mobj.group()
             if 'mp4' in video_url:
@@ -3816,7 +3501,7 @@ class WorldStarHipHopIE(InfoExtractor):
             else:
                 ext = 'flv'
         else:
-            self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id)
+            self._downloader.report_error(u'Cannot find video url for %s' % video_id)
             return
 
         _title = r"""<title>(.*)</title>"""
@@ -3928,7 +3613,7 @@ class YouPornIE(InfoExtractor):
             self._downloader.report_warning(u'unable to extract video date')
             upload_date = None
         else:
-            upload_date = result.group('date').strip()
+            upload_date = unified_strdate(result.group('date').strip())
 
         # Get the video uploader
         result = re.search(r'Submitted:</label>(?P<uploader>.*)</li>', webpage)
@@ -4035,7 +3720,7 @@ class PornotubeIE(InfoExtractor):
         if result is None:
             self._downloader.report_error(u'unable to extract video title')
             return
-        upload_date = result.group('date')
+        upload_date = unified_strdate(result.group('date'))
 
         info = {'id': video_id,
                 'url': video_url,
@@ -4162,12 +3847,13 @@ class KeekIE(InfoExtractor):
         return [info]
 
 class TEDIE(InfoExtractor):
-    _VALID_URL=r'''http://www.ted.com/
+    _VALID_URL=r'''http://www\.ted\.com/
                    (
                         ((?P<type_playlist>playlists)/(?P<playlist_id>\d+)) # We have a playlist
                         |
                         ((?P<type_talk>talks)) # We have a simple talk
                    )
+                   (/lang/(.*?))? # The url may contain the language
                    /(?P<name>\w+) # Here goes the name and then ".html"
                    '''
 
@@ -4336,7 +4022,7 @@ class LiveLeakIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
-            self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+            self._downloader.report_error(u'invalid URL: %s' % url)
             return
 
         video_id = mobj.group('video_id')
@@ -4351,7 +4037,7 @@ class LiveLeakIE(InfoExtractor):
 
         m = re.search(r'<meta property="og:title" content="(?P<title>.*?)"', webpage)
         if not m:
-            self._downloader.trouble(u'Cannot find video title')
+            self._downloader.report_error(u'Cannot find video title')
         title = unescapeHTML(m.group('title')).replace('LiveLeak.com -', '').strip()
 
         m = re.search(r'<meta property="og:description" content="(?P<desc>.*?)"', webpage)
@@ -4417,6 +4103,119 @@ class ARDIE(InfoExtractor):
             info["url"] = stream["video_url"]
         return [info]
 
+class TumblrIE(InfoExtractor):
+    _VALID_URL = r'http://(?P<blog_name>.*?)\.tumblr\.com/((post)|(video))/(?P<id>\d*)/(.*?)'
+
+    def _real_extract(self, url):
+        m_url = re.match(self._VALID_URL, url)
+        video_id = m_url.group('id')
+        blog = m_url.group('blog_name')
+
+        url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
+        webpage = self._download_webpage(url, video_id)
+
+        re_video = r'src=\\x22(?P<video_url>http://%s\.tumblr\.com/video_file/%s/(.*?))\\x22 type=\\x22video/(?P<ext>.*?)\\x22' % (blog, video_id)
+        video = re.search(re_video, webpage)
+        if video is None:
+            self.to_screen("No video founded")
+            return []
+        video_url = video.group('video_url')
+        ext = video.group('ext')
+
+        re_thumb = r'posters(.*?)\[\\x22(?P<thumb>.*?)\\x22'  # We pick the first poster
+        thumb = re.search(re_thumb, webpage).group('thumb').replace('\\', '')
+
+        # The only place where you can get a title, it's not complete,
+        # but searching in other places doesn't work for all videos
+        re_title = r'<title>(?P<title>.*?)</title>'
+        title = unescapeHTML(re.search(re_title, webpage, re.DOTALL).group('title'))
+
+        return [{'id': video_id,
+                 'url': video_url,
+                 'title': title,
+                 'thumbnail': thumb,
+                 'ext': ext
+                 }]
+
+class BandcampIE(InfoExtractor):
+    _VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P<title>.*)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        title = mobj.group('title')
+        webpage = self._download_webpage(url, title)
+        # We get the link to the free download page
+        m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage)
+        if m_download is None:
+            self._downloader.report_error('No free songs founded')
+            return
+        download_link = m_download.group(1)
+        id = re.search(r'var TralbumData = {(.*?)id: (?P<id>\d*?)$', 
+                       webpage, re.MULTILINE|re.DOTALL).group('id')
+
+        download_webpage = self._download_webpage(download_link, id,
+                                                  'Downloading free downloads page')
+        # We get the dictionary of the track from some javascrip code
+        info = re.search(r'items: (.*?),$',
+                         download_webpage, re.MULTILINE).group(1)
+        info = json.loads(info)[0]
+        # We pick mp3-320 for now, until format selection can be easily implemented.
+        mp3_info = info[u'downloads'][u'mp3-320']
+        # If we try to use this url it says the link has expired
+        initial_url = mp3_info[u'url']
+        re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
+        m_url = re.match(re_url, initial_url)
+        #We build the url we will use to get the final track url
+        # This url is build in Bandcamp in the script download_bunde_*.js
+        request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), id, m_url.group('ts'))
+        final_url_webpage = self._download_webpage(request_url, id, 'Requesting download url')
+        # If we could correctly generate the .rand field the url would be
+        #in the "download_url" key
+        final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
+
+        track_info = {'id':id,
+                      'title' : info[u'title'],
+                      'ext' : 'mp3',
+                      'url' : final_url,
+                      'thumbnail' : info[u'thumb_url'],
+                      'uploader' : info[u'artist']
+                      }
+
+        return [track_info]
+
+class RedtubeIE(InfoExtractor):
+    """Information Extractor for redtube"""
+    _VALID_URL = r'(?:http://)?(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
+    IE_NAME = u'redtube'
+
+    def _real_extract(self,url):
+        mobj = re.match(self._VALID_URL, url)
+        if mobj is None:
+            self._downloader.report_error(u'invalid URL: %s' % url)
+            return
+        video_id = mobj.group('id')
+        video_extension = 'mp4'        
+        webpage = self._download_webpage(url, video_id)
+        self.report_extraction(video_id)
+        mobj = re.search(r'<source src="'+'(.+)'+'" type="video/mp4">',webpage)
+        if mobj is not None:
+            video_url = mobj.group(1)
+        else:
+            self._downloader.report_error(u'unable to extract media URL')
+            return
+        mobj = re.search('<h1 class="videoTitle slidePanelMovable">'+r'(.+)'+r'</h1>',webpage)
+        if mobj is not None:
+            video_title = mobj.group(1)
+        else:
+            video_title = 'Redtube - %s' % time.ctime()
+
+        return [{
+            'id':       video_id,
+            'url':      video_url,
+            'ext':      video_extension,
+            'title':    video_title,
+        }]
+
 
 def gen_extractors():
     """ Return a list of an instance of every supported extractor.
@@ -4471,6 +4270,9 @@ def gen_extractors():
         SpiegelIE(),
         LiveLeakIE(),
         ARDIE(),
+        TumblrIE(),
+        BandcampIE(),
+        RedtubeIE(),
         GenericIE()
     ]