InfoQ IE (Closes #216)
authorPhilipp Hagemeister <phihag@phihag.de>
Tue, 15 Nov 2011 22:00:31 +0000 (23:00 +0100)
committerPhilipp Hagemeister <phihag@phihag.de>
Tue, 15 Nov 2011 22:00:31 +0000 (23:00 +0100)
1  2 
youtube-dl

diff --cc youtube-dl
index ff017759a763ce1a4627e39d189310f04aca3156,884e6f9d2dc87b7fcad96410d6c6d18ba3006a14..d271e1a04d6e44b2d75a158e7eedf72925c77dc8
@@@ -3482,107 -3482,91 +3482,193 @@@ class XVideosIE(InfoExtractor)
                        self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
  
  
 +class SoundcloudIE(InfoExtractor):
 +      """Information extractor for soundcloud.com
 +         To access the media, the uid of the song and a stream token
 +         must be extracted from the page source and the script must make
 +         a request to media.soundcloud.com/crossdomain.xml. Then
 +         the media can be grabbed by requesting from an url composed
 +         of the stream token and uid
 +       """
 +
 +      _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
 +      IE_NAME = u'soundcloud'
 +
 +      def __init__(self, downloader=None):
 +              InfoExtractor.__init__(self, downloader)
 +
 +      def report_webpage(self, video_id):
 +              """Report information extraction."""
 +              self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
 +
 +      def report_extraction(self, video_id):
 +              """Report information extraction."""
 +              self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
 +
 +      def _real_initialize(self):
 +              return
 +
 +      def _real_extract(self, url):
 +              htmlParser = HTMLParser.HTMLParser()
 +
 +              mobj = re.match(self._VALID_URL, url)
 +              if mobj is None:
 +                      self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 +                      return
 +
 +              # extract uploader (which is in the url)
 +              uploader = mobj.group(1).decode('utf-8')
 +              # extract simple title (uploader + slug of song title)
 +              slug_title =  mobj.group(2).decode('utf-8')
 +              simple_title = uploader + '-' + slug_title
 +
 +              self.report_webpage('%s/%s' % (uploader, slug_title))
 +
 +              request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
 +              try:
 +                      webpage = urllib2.urlopen(request).read()
 +              except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 +                      self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
 +                      return
 +
 +              self.report_extraction('%s/%s' % (uploader, slug_title))
 +
 +              # extract uid and stream token that soundcloud hands out for access
 +              mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)   
 +              if mobj:
 +                      video_id = mobj.group(1)
 +                      stream_token = mobj.group(2)
 +
 +              # extract unsimplified title
 +              mobj = re.search('"title":"(.*?)",', webpage)
 +              if mobj:
 +                      title = mobj.group(1)
 +
 +              # construct media url (with uid/token)
 +              mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
 +              mediaURL = mediaURL % (video_id, stream_token)
 +
 +              # description
 +              description = u'No description available'
 +              mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
 +              if mobj:
 +                      description = mobj.group(1)
 +              
 +              # upload date
 +              upload_date = None
 +              mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
 +              if mobj:
 +                      try:
 +                              upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
 +                      except Exception as e:
 +                              print str(e)
 +
 +              # for soundcloud, a request to a cross domain is required for cookies
 +              request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
 +
 +              try:
 +                      self._downloader.process_info({
 +                              'id':           video_id.decode('utf-8'),
 +                              'url':          mediaURL,
 +                              'uploader':     uploader.decode('utf-8'),
 +                              'upload_date':  upload_date,
 +                              'title':        simple_title.decode('utf-8'),
 +                              'stitle':       simple_title.decode('utf-8'),
 +                              'ext':          u'mp3',
 +                              'format':       u'NA',
 +                              'player_url':   None,
 +                              'description': description.decode('utf-8')
 +                      })
 +              except UnavailableVideoError:
 +                      self._downloader.trouble(u'\nERROR: unable to download video')
 +
 +
+ class InfoQIE(InfoExtractor):
+       """Information extractor for infoq.com"""
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
+       IE_NAME = u'infoq'
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+       def _simplify_title(self, title):
+               res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+               res = res.strip(ur'_')
+               return res
+       def _real_extract(self, url):
+               htmlParser = HTMLParser.HTMLParser()
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               self.report_webpage(url)
+               request = urllib2.Request(url)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+               self.report_extraction(url)
+               # Extract video URL
+               mobj = re.search(r"jsclassref='([^']*)'", webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video url')
+                       return
+               video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
+               # Extract title
+               mobj = re.search(r'contentTitle = "(.*?)";', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video title')
+                       return
+               video_title = mobj.group(1).decode('utf-8')
+               # Extract description
+               video_description = u'No description available.'
+               mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
+               if mobj is not None:
+                       video_description = mobj.group(1).decode('utf-8')
+               video_filename = video_url.split('/')[-1]
+               video_id, extension = video_filename.split('.')
+               self._downloader.increment_downloads()
+               info = {
+                       'id': video_id,
+                       'url': video_url,
+                       'uploader': None,
+                       'upload_date': None,
+                       'title': video_title,
+                       'stitle': self._simplify_title(video_title),
+                       'ext': extension,
+                       'format': extension, # Extension is always(?) mp4, but seems to be flv
+                       'thumbnail': None,
+                       'description': video_description,
+                       'player_url': None,
+               }
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download ' + video_url)
++
  class PostProcessor(object):
        """Post Processor class.
  
@@@ -3979,7 -3963,7 +4065,8 @@@ def gen_extractors()
                EscapistIE(),
                CollegeHumorIE(),
                XVideosIE(),
 +              SoundcloudIE(),
+               InfoQIE(),
  
                GenericIE()
        ]