Merge pull request #472 from gcmalloc/master

[youtube-dl] / youtube_dl / InfoExtractors.py

diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index f97611cb9b4ab681eb21f760db4c71ea1b2da98a..88973cce8982b70a57039a26208a9c647474160f 100644 (file)
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -658,15 +658,21 @@ class DailymotionIE(InfoExtractor):
                        self._downloader.trouble(u'ERROR: unable to extract media URL')
                        return
                flashvars = urllib.unquote(mobj.group(1))
-               if 'hqURL' in flashvars: max_quality = 'hqURL'
-               elif 'sdURL' in flashvars: max_quality = 'sdURL'
-               else: max_quality = 'ldURL'
+
+               for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
+                       if key in flashvars:
+                               max_quality = key
+                               self._downloader.to_screen(u'[dailymotion] Using %s' % key)
+                               break
+               else:
+                       self._downloader.trouble(u'ERROR: unable to extract video URL')
+                       return
+
                mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
                if mobj is None:
-                       mobj = re.search(r'"video_url":"(.*?)",', flashvars)
-               if mobj is None:
-                       self._downloader.trouble(u'ERROR: unable to extract media URL')
+                       self._downloader.trouble(u'ERROR: unable to extract video URL')
                        return
+
                video_url = urllib.unquote(mobj.group(1)).replace('\\/', '/')
 
                # TODO: support choosing qualities
@@ -677,11 +683,12 @@ class DailymotionIE(InfoExtractor):
                        return
                video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
 
+               video_uploader = u'NA'
                mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
                if mobj is None:
-                       self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
-                       return
-               video_uploader = mobj.group(1)
+                       self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
+               else:
+                       video_uploader = mobj.group(1)
 
                video_upload_date = u'NA'
                mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
@@ -1501,9 +1508,9 @@ class YahooSearchIE(InfoExtractor):
 class YoutubePlaylistIE(InfoExtractor):
        """Information Extractor for YouTube playlists."""
 
-       _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+       _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
        _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
-       _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;list=.*?%s'
+       _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
        _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
        IE_NAME = u'youtube:playlist'
 
@@ -1571,6 +1578,56 @@ class YoutubePlaylistIE(InfoExtractor):
                return
 
 
+class YoutubeChannelIE(InfoExtractor):
+       """Information Extractor for YouTube channels."""
+
+       _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
+       _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
+       _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
+       IE_NAME = u'youtube:channel'
+
+       def report_download_page(self, channel_id, pagenum):
+               """Report attempt to download channel page with given number."""
+               self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
+
+       def _real_extract(self, url):
+               # Extract channel id
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid url: %s' % url)
+                       return
+
+               # Download channel pages
+               channel_id = mobj.group(1)
+               video_ids = []
+               pagenum = 1
+
+               while True:
+                       self.report_download_page(channel_id, pagenum)
+                       url = self._TEMPLATE_URL % (channel_id, pagenum)
+                       request = urllib2.Request(url)
+                       try:
+                               page = urllib2.urlopen(request).read()
+                       except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                               self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+                               return
+
+                       # Extract video identifiers
+                       ids_in_page = []
+                       for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
+                               if mobj.group(1) not in ids_in_page:
+                                       ids_in_page.append(mobj.group(1))
+                       video_ids.extend(ids_in_page)
+
+                       if re.search(self._MORE_PAGES_INDICATOR, page) is None:
+                               break
+                       pagenum = pagenum + 1
+
+               for id in video_ids:
+                       self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
+               return
+
+
 class YoutubeUserIE(InfoExtractor):
        """Information Extractor for YouTube users."""