X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;ds=sidebyside;f=youtube-dl;h=fe23e9f8f036eadb9c8ba711a08144bee93d459f;hb=2fb47e073a1e0c676130ef0c62e9949dd40af8ed;hp=5a68a2ee9a169039b4ced962d12b5511af7844d7;hpb=c4cfbdf5a5ce8203aea55d27a4a3f0e1b495fafc;p=youtube-dl diff --git a/youtube-dl b/youtube-dl index 5a68a2ee9..fe23e9f8f 100755 --- a/youtube-dl +++ b/youtube-dl @@ -38,7 +38,7 @@ except ImportError: from cgi import parse_qs std_headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:2.0b10) Gecko/20100101 Firefox/4.0b10', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate', @@ -858,7 +858,7 @@ class InfoExtractor(object): class YoutubeIE(InfoExtractor): """Information extractor for youtube.com.""" - _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$' + _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$' _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en' _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' @@ -1056,10 +1056,10 @@ class YoutubeIE(InfoExtractor): # upload date upload_date = u'NA' - mobj = re.search(r'id="eow-date".*?>(.*?)', video_webpage, re.DOTALL) + mobj = re.search(r'id="eow-date.*?>(.*?)', video_webpage, re.DOTALL) if mobj is not None: upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split()) - format_expressions = ['%d %B %Y', '%B %d %Y'] + format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y'] for expression in format_expressions: try: upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d') @@ -1079,8 +1079,10 @@ class YoutubeIE(InfoExtractor): # Decide which formats to download req_format = self._downloader.params.get('format', None) - if 'fmt_url_map' in video_info: - url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(',')) + if 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1: + url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',') + url_data = [dict(pairStr.split('=') for pairStr in uds.split('&')) for uds in url_data_strs] + url_map = dict((ud['itag'], urllib.unquote(ud['url'])) for ud in url_data) format_limit = self._downloader.params.get('format_limit', None) if format_limit is not None and format_limit in self._available_formats: format_list = self._available_formats[self._available_formats.index(format_limit):] @@ -1724,7 +1726,7 @@ class VimeoIE(InfoExtractor): """Information extractor for vimeo.com.""" # _VALID_URL matches Vimeo URLs - _VALID_URL = r'(?:http://)?vimeo\.com/([0-9]+)' + _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)' def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) @@ -1735,11 +1737,11 @@ class VimeoIE(InfoExtractor): def report_download_webpage(self, video_id): """Report webpage download.""" - self._downloader.to_screen(u'[video.vimeo] %s: Downloading webpage' % video_id) + self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id) def report_extraction(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[video.vimeo] %s: Extracting information' % video_id) + self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id) def _real_initialize(self): return @@ -1754,7 +1756,6 @@ class VimeoIE(InfoExtractor): # At this point we have a new video self._downloader.increment_downloads() video_id = mobj.group(1) - video_extension = 'flv' # FIXME # Retrieve video webpage to extract further information request = urllib2.Request("http://vimeo.com/moogaloop/load/clip:%s" % video_id, None, std_headers) @@ -1765,8 +1766,12 @@ class VimeoIE(InfoExtractor): self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) return - # Extract uploader and title from webpage + # Now we begin extracting as much information as we can from what we + # retrieved. First we extract the information common to all extractors, + # and latter we extract those that are Vimeo specific. self.report_extraction(video_id) + + # Extract title mobj = re.search(r'(.*?)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video title') @@ -1774,6 +1779,7 @@ class VimeoIE(InfoExtractor): video_title = mobj.group(1).decode('utf-8') simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title) + # Extract uploader mobj = re.search(r'http://vimeo.com/(.*?)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract video uploader') @@ -1796,14 +1802,14 @@ class VimeoIE(InfoExtractor): # if not video_description: video_description = 'No description available.' video_description = 'Foo.' - # Extract request signature + # Vimeo specific: extract request signature mobj = re.search(r'(.*?)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract request signature') return sig = mobj.group(1).decode('utf-8') - # Extract request signature expiration + # Vimeo specific: Extract request signature expiration mobj = re.search(r'(.*?)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract request signature expiration') @@ -1821,7 +1827,7 @@ class VimeoIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), + 'ext': u'mp4', 'thumbnail': video_thumbnail.decode('utf-8'), 'description': video_description, 'thumbnail': video_thumbnail, @@ -2721,6 +2727,88 @@ class PostProcessor(object): """ return information # by default, do nothing +class FFmpegExtractAudioPP(PostProcessor): + + def __init__(self, downloader=None, preferredcodec=None): + PostProcessor.__init__(self, downloader) + if preferredcodec is None: + preferredcodec = 'best' + self._preferredcodec = preferredcodec + + @staticmethod + def get_audio_codec(path): + try: + cmd = ['ffprobe', '-show_streams', '--', path] + handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE) + output = handle.communicate()[0] + if handle.wait() != 0: + return None + except (IOError, OSError): + return None + audio_codec = None + for line in output.split('\n'): + if line.startswith('codec_name='): + audio_codec = line.split('=')[1].strip() + elif line.strip() == 'codec_type=audio' and audio_codec is not None: + return audio_codec + return None + + @staticmethod + def run_ffmpeg(path, out_path, codec, more_opts): + try: + cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path] + ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT) + return (ret == 0) + except (IOError, OSError): + return False + + def run(self, information): + path = information['filepath'] + + filecodec = self.get_audio_codec(path) + if filecodec is None: + self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe') + return None + + more_opts = [] + if self._preferredcodec == 'best' or self._preferredcodec == filecodec: + if filecodec == 'aac' or filecodec == 'mp3': + # Lossless if possible + acodec = 'copy' + extension = filecodec + if filecodec == 'aac': + more_opts = ['-f', 'adts'] + else: + # MP3 otherwise. + acodec = 'libmp3lame' + extension = 'mp3' + more_opts = ['-ab', '128k'] + else: + # We convert the audio (lossy) + acodec = {'mp3': 'libmp3lame', 'aac': 'aac'}[self._preferredcodec] + extension = self._preferredcodec + more_opts = ['-ab', '128k'] + if self._preferredcodec == 'aac': + more_opts += ['-f', 'adts'] + + (prefix, ext) = os.path.splitext(path) + new_path = prefix + '.' + extension + self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path) + status = self.run_ffmpeg(path, new_path, acodec, more_opts) + + if not status: + self._downloader.to_stderr(u'WARNING: error running ffmpeg') + return None + + try: + os.remove(path) + except (IOError, OSError): + self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') + return None + + information['filepath'] = new_path + return information + ### MAIN PROGRAM ### if __name__ == '__main__': try: @@ -2753,7 +2841,7 @@ if __name__ == '__main__': # Parse command line parser = optparse.OptionParser( usage='Usage: %prog [options] url...', - version='2011.01.30', + version='2011.08.04', conflict_handler='resolve', ) @@ -2845,6 +2933,13 @@ if __name__ == '__main__': help='do not use the Last-modified header to set the file modification time', default=True) parser.add_option_group(filesystem) + postproc = optparse.OptionGroup(parser, 'Post-processing Options') + postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False, + help='convert video files to audio-only files (requires ffmpeg and ffprobe)') + postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', + help='"best", "aac" or "mp3"; best by default') + parser.add_option_group(postproc) + (opts, args) = parser.parse_args() # Open appropriate CookieJar @@ -2916,6 +3011,9 @@ if __name__ == '__main__': raise ValueError except (TypeError, ValueError), err: parser.error(u'invalid playlist end number specified') + if opts.extractaudio: + if opts.audioformat not in ['best', 'aac', 'mp3']: + parser.error(u'invalid audio format specified') # Information extractors vimeo_ie = VimeoIE() @@ -2990,6 +3088,10 @@ if __name__ == '__main__': # fallback if none of the others work fd.add_info_extractor(generic_ie) + # PostProcessors + if opts.extractaudio: + fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat)) + # Update version if opts.update_self: update_self(fd, sys.argv[0])