+ _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
+ IE_NAME = u'comedycentral'
+
+ def report_extraction(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
+
+ def report_config_download(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
+
+ def report_index_download(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
+
+ def report_player_url(self, episode_id):
+ self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ if mobj.group('shortname'):
+ if mobj.group('shortname') in ('tds', 'thedailyshow'):
+ url = u'http://www.thedailyshow.com/full-episodes/'
+ else:
+ url = u'http://www.colbertnation.com/full-episodes/'
+ mobj = re.match(self._VALID_URL, url)
+ assert mobj is not None
+
+ dlNewest = not mobj.group('episode')
+ if dlNewest:
+ epTitle = mobj.group('showname')
+ else:
+ epTitle = mobj.group('episode')
+
+ req = urllib2.Request(url)
+ self.report_extraction(epTitle)
+ try:
+ htmlHandle = urllib2.urlopen(req)
+ html = htmlHandle.read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+ return
+ if dlNewest:
+ url = htmlHandle.geturl()
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
+ return
+ if mobj.group('episode') == '':
+ self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
+ return
+ epTitle = mobj.group('episode')
+
+ mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"/>', html)
+ if len(mMovieParams) == 0:
+ self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
+ return
+
+ playerUrl_raw = mMovieParams[0][0]
+ self.report_player_url(epTitle)
+ try:
+ urlHandle = urllib2.urlopen(playerUrl_raw)
+ playerUrl = urlHandle.geturl()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
+ return
+
+ uri = mMovieParams[0][1]
+ indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
+ self.report_index_download(epTitle)
+ try:
+ indexXml = urllib2.urlopen(indexUrl).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
+ return
+
+ idoc = xml.etree.ElementTree.fromstring(indexXml)
+ itemEls = idoc.findall('.//item')
+ for itemEl in itemEls:
+ mediaId = itemEl.findall('./guid')[0].text
+ shortMediaId = mediaId.split(':')[-1]
+ showId = mediaId.split(':')[-2].replace('.com', '')
+ officialTitle = itemEl.findall('./title')[0].text
+ officialDate = itemEl.findall('./pubDate')[0].text
+
+ configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
+ urllib.urlencode({'uri': mediaId}))
+ configReq = urllib2.Request(configUrl)
+ self.report_config_download(epTitle)
+ try:
+ configXml = urllib2.urlopen(configReq).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+ return
+
+ cdoc = xml.etree.ElementTree.fromstring(configXml)
+ turls = []
+ for rendition in cdoc.findall('.//rendition'):
+ finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
+ turls.append(finfo)
+
+ if len(turls) == 0:
+ self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
+ continue
+
+ # For now, just pick the highest bitrate
+ format,video_url = turls[-1]
+
+ self._downloader.increment_downloads()
+
+ effTitle = showId + u'-' + epTitle
+ info = {
+ 'id': shortMediaId,
+ 'url': video_url,
+ 'uploader': showId,
+ 'upload_date': officialDate,
+ 'title': effTitle,
+ 'stitle': _simplify_title(effTitle),
+ 'ext': 'mp4',
+ 'format': format,
+ 'thumbnail': None,
+ 'description': officialTitle,
+ 'player_url': playerUrl
+ }
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download ' + mediaId)
+ continue
+
+
+class EscapistIE(InfoExtractor):
+ """Information extractor for The Escapist """
+
+ _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
+ IE_NAME = u'escapist'
+
+ def report_extraction(self, showName):
+ self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
+
+ def report_config_download(self, showName):
+ self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
+
+ def _real_extract(self, url):
+ htmlParser = HTMLParser.HTMLParser()
+
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ showName = mobj.group('showname')
+ videoId = mobj.group('episode')
+
+ self.report_extraction(showName)
+ try:
+ webPage = urllib2.urlopen(url).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
+ return
+
+ descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
+ description = htmlParser.unescape(descMatch.group(1))
+ imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
+ imgUrl = htmlParser.unescape(imgMatch.group(1))
+ playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
+ playerUrl = htmlParser.unescape(playerUrlMatch.group(1))
+ configUrlMatch = re.search('config=(.*)$', playerUrl)
+ configUrl = urllib2.unquote(configUrlMatch.group(1))
+
+ self.report_config_download(showName)
+ try:
+ configJSON = urllib2.urlopen(configUrl).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download configuration: ' + unicode(err))
+ return
+
+ # Technically, it's JavaScript, not JSON
+ configJSON = configJSON.replace("'", '"')
+
+ try:
+ config = json.loads(configJSON)
+ except (ValueError,), err:
+ self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + unicode(err))
+ return
+
+ playlist = config['playlist']
+ videoUrl = playlist[1]['url']
+
+ self._downloader.increment_downloads()
+ info = {
+ 'id': videoId,
+ 'url': videoUrl,
+ 'uploader': showName,
+ 'upload_date': None,
+ 'title': showName,
+ 'stitle': _simplify_title(showName),
+ 'ext': 'flv',
+ 'format': 'flv',
+ 'thumbnail': imgUrl,
+ 'description': description,
+ 'player_url': playerUrl,
+ }
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
+
+
+class CollegeHumorIE(InfoExtractor):
+ """Information extractor for collegehumor.com"""
+
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
+ IE_NAME = u'collegehumor'
+
+ def report_webpage(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ htmlParser = HTMLParser.HTMLParser()
+
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ video_id = mobj.group('videoid')
+
+ self.report_webpage(video_id)
+ request = urllib2.Request(url)
+ try:
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+
+ m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
+ if m is None:
+ self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
+ return
+ internal_video_id = m.group('internalvideoid')
+
+ info = {
+ 'id': video_id,
+ 'internal_id': internal_video_id,
+ }
+
+ self.report_extraction(video_id)
+ xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
+ try:
+ metaXml = urllib2.urlopen(xmlUrl).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
+ return
+
+ mdoc = xml.etree.ElementTree.fromstring(metaXml)
+ try:
+ videoNode = mdoc.findall('./video')[0]
+ info['description'] = videoNode.findall('./description')[0].text
+ info['title'] = videoNode.findall('./caption')[0].text
+ info['stitle'] = _simplify_title(info['title'])
+ info['url'] = videoNode.findall('./file')[0].text
+ info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
+ info['ext'] = info['url'].rpartition('.')[2]
+ info['format'] = info['ext']
+ except IndexError:
+ self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+ return
+
+ self._downloader.increment_downloads()
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class XVideosIE(InfoExtractor):
+ """Information extractor for xvideos.com"""
+
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
+ IE_NAME = u'xvideos'
+
+ def report_webpage(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ htmlParser = HTMLParser.HTMLParser()
+
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ video_id = mobj.group(1).decode('utf-8')
+
+ self.report_webpage(video_id)
+
+ request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
+ try:
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+
+ self.report_extraction(video_id)
+
+
+ # Extract video URL
+ mobj = re.search(r'flv_url=(.+?)&', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video url')
+ return
+ video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
+
+
+ # Extract title
+ mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video title')
+ return
+ video_title = mobj.group(1).decode('utf-8')
+
+
+ # Extract video thumbnail
+ mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
+ return
+ video_thumbnail = mobj.group(1).decode('utf-8')
+
+
+
+ self._downloader.increment_downloads()
+ info = {
+ 'id': video_id,
+ 'url': video_url,
+ 'uploader': None,
+ 'upload_date': None,
+ 'title': video_title,
+ 'stitle': _simplify_title(video_title),
+ 'ext': 'flv',
+ 'format': 'flv',
+ 'thumbnail': video_thumbnail,
+ 'description': None,
+ 'player_url': None,
+ }
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+
+
+class SoundcloudIE(InfoExtractor):
+ """Information extractor for soundcloud.com
+ To access the media, the uid of the song and a stream token
+ must be extracted from the page source and the script must make
+ a request to media.soundcloud.com/crossdomain.xml. Then
+ the media can be grabbed by requesting from an url composed
+ of the stream token and uid
+ """
+
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
+ IE_NAME = u'soundcloud'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ def report_webpage(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ htmlParser = HTMLParser.HTMLParser()
+
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ # extract uploader (which is in the url)
+ uploader = mobj.group(1).decode('utf-8')
+ # extract simple title (uploader + slug of song title)
+ slug_title = mobj.group(2).decode('utf-8')
+ simple_title = uploader + '-' + slug_title
+
+ self.report_webpage('%s/%s' % (uploader, slug_title))
+
+ request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+ try:
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+
+ self.report_extraction('%s/%s' % (uploader, slug_title))
+
+ # extract uid and stream token that soundcloud hands out for access
+ mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
+ if mobj:
+ video_id = mobj.group(1)
+ stream_token = mobj.group(2)
+
+ # extract unsimplified title
+ mobj = re.search('"title":"(.*?)",', webpage)
+ if mobj:
+ title = mobj.group(1)
+
+ # construct media url (with uid/token)
+ mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
+ mediaURL = mediaURL % (video_id, stream_token)
+
+ # description
+ description = u'No description available'
+ mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
+ if mobj:
+ description = mobj.group(1)
+
+ # upload date
+ upload_date = None
+ mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
+ if mobj:
+ try:
+ upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
+ except Exception, e:
+ print str(e)
+
+ # for soundcloud, a request to a cross domain is required for cookies
+ request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+
+ try:
+ self._downloader.process_info({
+ 'id': video_id.decode('utf-8'),
+ 'url': mediaURL,
+ 'uploader': uploader.decode('utf-8'),
+ 'upload_date': upload_date,
+ 'title': simple_title.decode('utf-8'),
+ 'stitle': simple_title.decode('utf-8'),
+ 'ext': u'mp3',
+ 'format': u'NA',
+ 'player_url': None,
+ 'description': description.decode('utf-8')
+ })
+ except UnavailableVideoError:
+ self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class InfoQIE(InfoExtractor):
+ """Information extractor for infoq.com"""
+
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
+ IE_NAME = u'infoq'
+
+ def report_webpage(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ htmlParser = HTMLParser.HTMLParser()
+
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ self.report_webpage(url)
+
+ request = urllib2.Request(url)
+ try:
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+
+ self.report_extraction(url)
+
+
+ # Extract video URL
+ mobj = re.search(r"jsclassref='([^']*)'", webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video url')
+ return
+ video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
+
+
+ # Extract title
+ mobj = re.search(r'contentTitle = "(.*?)";', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video title')
+ return
+ video_title = mobj.group(1).decode('utf-8')
+
+ # Extract description
+ video_description = u'No description available.'
+ mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
+ if mobj is not None:
+ video_description = mobj.group(1).decode('utf-8')
+
+ video_filename = video_url.split('/')[-1]
+ video_id, extension = video_filename.split('.')
+
+ self._downloader.increment_downloads()
+ info = {
+ 'id': video_id,
+ 'url': video_url,
+ 'uploader': None,
+ 'upload_date': None,
+ 'title': video_title,
+ 'stitle': _simplify_title(video_title),
+ 'ext': extension,
+ 'format': extension, # Extension is always(?) mp4, but seems to be flv
+ 'thumbnail': None,
+ 'description': video_description,
+ 'player_url': None,
+ }
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download ' + video_url)
+
+class MixcloudIE(InfoExtractor):
+ """Information extractor for www.mixcloud.com"""
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
+ IE_NAME = u'mixcloud'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ def report_download_json(self, file_id):
+ """Report JSON download."""
+ self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
+
+ def report_extraction(self, file_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
+
+ def get_urls(self, jsonData, fmt, bitrate='best'):
+ """Get urls from 'audio_formats' section in json"""
+ file_url = None
+ try:
+ bitrate_list = jsonData[fmt]
+ if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
+ bitrate = max(bitrate_list) # select highest
+
+ url_list = jsonData[fmt][bitrate]
+ except TypeError: # we have no bitrate info.
+ url_list = jsonData[fmt]
+
+ return url_list
+
+ def check_urls(self, url_list):
+ """Returns 1st active url from list"""
+ for url in url_list:
+ try:
+ urllib2.urlopen(url)
+ return url
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ url = None
+
+ return None
+
+ def _print_formats(self, formats):
+ print 'Available formats:'
+ for fmt in formats.keys():
+ for b in formats[fmt]:
+ try:
+ ext = formats[fmt][b][0]
+ print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])
+ except TypeError: # we have no bitrate info
+ ext = formats[fmt][0]
+ print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])
+ break
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ # extract uploader & filename from url
+ uploader = mobj.group(1).decode('utf-8')
+ file_id = uploader + "-" + mobj.group(2).decode('utf-8')
+
+ # construct API request
+ file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
+ # retrieve .json file with links to files
+ request = urllib2.Request(file_url)
+ try:
+ self.report_download_json(file_url)
+ jsonData = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err))
+ return
+
+ # parse JSON
+ json_data = json.loads(jsonData)
+ player_url = json_data['player_swf_url']
+ formats = dict(json_data['audio_formats'])
+
+ req_format = self._downloader.params.get('format', None)
+ bitrate = None
+
+ if self._downloader.params.get('listformats', None):
+ self._print_formats(formats)
+ return
+
+ if req_format is None or req_format == 'best':
+ for format_param in formats.keys():
+ url_list = self.get_urls(formats, format_param)
+ # check urls
+ file_url = self.check_urls(url_list)
+ if file_url is not None:
+ break # got it!
+ else:
+ if req_format not in formats.keys():
+ self._downloader.trouble(u'ERROR: format is not available')
+ return
+
+ url_list = self.get_urls(formats, req_format)
+ file_url = self.check_urls(url_list)
+ format_param = req_format
+
+ # We have audio
+ self._downloader.increment_downloads()
+ try:
+ # Process file information
+ self._downloader.process_info({
+ 'id': file_id.decode('utf-8'),
+ 'url': file_url.decode('utf-8'),
+ 'uploader': uploader.decode('utf-8'),
+ 'upload_date': u'NA',
+ 'title': json_data['name'],
+ 'stitle': _simplify_title(json_data['name']),
+ 'ext': file_url.split('.')[-1].decode('utf-8'),
+ 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
+ 'thumbnail': json_data['thumbnail_url'],
+ 'description': json_data['description'],
+ 'player_url': player_url.decode('utf-8'),
+ })
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'ERROR: unable to download file')
+
+class StanfordOpenClassroomIE(InfoExtractor):
+ """Information extractor for Stanford's Open ClassRoom"""
+
+ _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
+ IE_NAME = u'stanfordoc'
+
+ def report_download_webpage(self, objid):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ if mobj.group('course') and mobj.group('video'): # A specific video
+ course = mobj.group('course')
+ video = mobj.group('video')
+ info = {
+ 'id': _simplify_title(course + '_' + video),
+ }
+
+ self.report_extraction(info['id'])
+ baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
+ xmlUrl = baseUrl + video + '.xml'
+ try:
+ metaXml = urllib2.urlopen(xmlUrl).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
+ return
+ mdoc = xml.etree.ElementTree.fromstring(metaXml)
+ try:
+ info['title'] = mdoc.findall('./title')[0].text
+ info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
+ except IndexError:
+ self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+ return
+ info['stitle'] = _simplify_title(info['title'])
+ info['ext'] = info['url'].rpartition('.')[2]
+ info['format'] = info['ext']
+ self._downloader.increment_downloads()
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download video')
+ elif mobj.group('course'): # A course page
+ unescapeHTML = HTMLParser.HTMLParser().unescape
+
+ course = mobj.group('course')
+ info = {
+ 'id': _simplify_title(course),
+ 'type': 'playlist',
+ }
+
+ self.report_download_webpage(info['id'])
+ try:
+ coursepage = urllib2.urlopen(url).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+ return
+
+ m = re.search('<h1>([^<]+)</h1>', coursepage)
+ if m:
+ info['title'] = unescapeHTML(m.group(1))
+ else:
+ info['title'] = info['id']
+ info['stitle'] = _simplify_title(info['title'])
+
+ m = re.search('<description>([^<]+)</description>', coursepage)
+ if m:
+ info['description'] = unescapeHTML(m.group(1))
+
+ links = _orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
+ info['list'] = [
+ {
+ 'type': 'reference',
+ 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
+ }
+ for vpage in links]
+
+ for entry in info['list']:
+ assert entry['type'] == 'reference'
+ self.extract(entry['url'])
+ else: # Root page
+ unescapeHTML = HTMLParser.HTMLParser().unescape
+
+ info = {
+ 'id': 'Stanford OpenClassroom',
+ 'type': 'playlist',
+ }
+
+ self.report_download_webpage(info['id'])
+ rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
+ try:
+ rootpage = urllib2.urlopen(rootURL).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+ return
+
+ info['title'] = info['id']
+ info['stitle'] = _simplify_title(info['title'])
+
+ links = _orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
+ info['list'] = [
+ {
+ 'type': 'reference',
+ 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
+ }
+ for cpage in links]
+
+ for entry in info['list']:
+ assert entry['type'] == 'reference'
+ self.extract(entry['url'])
+
+
+class PostProcessor(object):
+ """Post Processor class.
+
+ PostProcessor objects can be added to downloaders with their
+ add_post_processor() method. When the downloader has finished a
+ successful download, it will take its internal chain of PostProcessors
+ and start calling the run() method on each one of them, first with
+ an initial argument and then with the returned value of the previous
+ PostProcessor.
+
+ The chain will be stopped if one of them ever returns None or the end
+ of the chain is reached.
+
+ PostProcessor objects follow a "mutual registration" process similar
+ to InfoExtractor objects.
+ """
+
+ _downloader = None
+
+ def __init__(self, downloader=None):
+ self._downloader = downloader
+
+ def set_downloader(self, downloader):
+ """Sets the downloader for this PP."""
+ self._downloader = downloader
+
+ def run(self, information):
+ """Run the PostProcessor.
+
+ The "information" argument is a dictionary like the ones
+ composed by InfoExtractors. The only difference is that this
+ one has an extra field called "filepath" that points to the
+ downloaded file.
+
+ When this method returns None, the postprocessing chain is
+ stopped. However, this method may return an information
+ dictionary that will be passed to the next postprocessing
+ object in the chain. It can be the one it received after
+ changing some fields.
+
+ In addition, this method may raise a PostProcessingError
+ exception that will be taken into account by the downloader
+ it was called from.
+ """
+ return information # by default, do nothing
+
+class AudioConversionError(BaseException):
+ def __init__(self, message):
+ self.message = message
+
+class FFmpegExtractAudioPP(PostProcessor):
+
+ def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
+ PostProcessor.__init__(self, downloader)
+ if preferredcodec is None:
+ preferredcodec = 'best'
+ self._preferredcodec = preferredcodec
+ self._preferredquality = preferredquality
+ self._keepvideo = keepvideo
+
+ @staticmethod
+ def get_audio_codec(path):
+ try:
+ cmd = ['ffprobe', '-show_streams', '--', path]
+ handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
+ output = handle.communicate()[0]
+ if handle.wait() != 0:
+ return None
projects / youtube-dl / blobdiff