From: Philipp Hagemeister Date: Mon, 13 May 2013 07:42:24 +0000 (-0700) Subject: Merge pull request #834 from chocolateboy/install_prefix_fix X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=commitdiff_plain;h=c1d58e1c67106e2f65fdd38fc4db99af19a647ce;hp=5738240ee829684815bdb9e6a9028d3465140b48;p=youtube-dl Merge pull request #834 from chocolateboy/install_prefix_fix only install to /etc if PREFIX is /usr or /usr/local --- diff --git a/README.md b/README.md index a9eaed192..2f3c81a7c 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ which means you can modify it, redistribute it or use it however you like. -F, --list-formats list all available formats (currently youtube only) --write-sub write subtitle file (currently youtube only) - --only-sub downloads only the subtitles (no video) + --only-sub [deprecated] alias of --skip-download --all-subs downloads all the available subtitles of the video (currently youtube only) --list-subs lists all available subtitles for the video diff --git a/test/tests.json b/test/tests.json index 5a14b0523..f0e5cd8bf 100644 --- a/test/tests.json +++ b/test/tests.json @@ -430,5 +430,15 @@ "info_dict":{ "title":"François Hollande \"Je crois que c'est clair\"" } + }, + { + "name": "Yahoo", + "url": "http://screen.yahoo.com/obama-celebrates-iraq-victory-27592561.html", + "file": "27592561.flv", + "md5": "c6179bed843512823fd284fa2e7f012d", + "info_dict": { + "title": "Obama Celebrates Iraq Victory" + }, + "skip": "Requires rtmpdump" } ] diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index eb68d9478..8a3bdf21b 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -83,7 +83,6 @@ class FileDownloader(object): writeinfojson: Write the video description to a .info.json file writethumbnail: Write the thumbnail image to a file writesubtitles: Write the video subtitles to a file - onlysubtitles: Downloads only the subtitles of the video allsubtitles: Downloads all the subtitles of the video listsubtitles: Lists all available subtitles for the video subtitlesformat: Subtitle format [sbv/srt] (default=srt) @@ -93,6 +92,7 @@ class FileDownloader(object): min_filesize: Skip files smaller than this size max_filesize: Skip files larger than this size daterange: A DateRange object, download only if the upload_date is in the range. + skip_download: Skip the actual download of the video file """ params = None @@ -597,7 +597,7 @@ class FileDownloader(object): try: dn = os.path.dirname(encodeFilename(filename)) - if dn != '' and not os.path.exists(dn): # dn is already encoded + if dn != '' and not os.path.exists(dn): os.makedirs(dn) except (OSError, IOError) as err: self.report_error(u'unable to create directory ' + compat_str(err)) @@ -630,8 +630,6 @@ class FileDownloader(object): except (OSError, IOError): self.report_error(u'Cannot write subtitles file ' + descfn) return - if self.params.get('onlysubtitles', False): - return if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: subtitles = info_dict['subtitles'] @@ -649,8 +647,6 @@ class FileDownloader(object): except (OSError, IOError): self.report_error(u'Cannot write subtitles file ' + descfn) return - if self.params.get('onlysubtitles', False): - return if self.params.get('writeinfojson', False): infofn = filename + u'.info.json' diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index ec52cbcff..4bcd8b9cd 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -912,123 +912,72 @@ class PhotobucketIE(InfoExtractor): class YahooIE(InfoExtractor): - """Information extractor for video.yahoo.com.""" + """Information extractor for screen.yahoo.com.""" + _VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P\d*?)\.html' - _WORKING = False - # _VALID_URL matches all Yahoo! Video URLs - # _VPAGE_URL matches only the extractable '/watch/' URLs - _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?' - _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?' - IE_NAME = u'video.yahoo' - - def _real_extract(self, url, new_video=True): - # Extract ID from URL + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj is None: raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group(2) - video_extension = 'flv' - - # Rewrite valid but non-extractable URLs as - # extractable English language /watch/ URLs - if re.match(self._VPAGE_URL, url) is None: - request = compat_urllib_request.Request(url) - try: - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to retrieve video webpage: %s' % compat_str(err)) - - mobj = re.search(r'\("id", "([0-9]+)"\);', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract id field') - yahoo_id = mobj.group(1) - - mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract vid field') - yahoo_vid = mobj.group(1) - - url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id) - return self._real_extract(url, new_video=False) - - # Retrieve video webpage to extract further information - request = compat_urllib_request.Request(url) - try: - self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to retrieve video webpage: %s' % compat_str(err)) - - # Extract uploader and title from webpage - self.report_extraction(video_id) - mobj = re.search(r'', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract video title') - video_title = mobj.group(1).decode('utf-8') - - mobj = re.search(r'

(.*)

', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract video uploader') - video_uploader = mobj.group(1).decode('utf-8') - - # Extract video thumbnail - mobj = re.search(r'', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract video thumbnail') - video_thumbnail = mobj.group(1).decode('utf-8') - - # Extract video description - mobj = re.search(r'', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract video description') - video_description = mobj.group(1).decode('utf-8') - if not video_description: - video_description = 'No description available.' - - # Extract video height and width - mobj = re.search(r'', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract video height') - yv_video_height = mobj.group(1) - - mobj = re.search(r'', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract video width') - yv_video_width = mobj.group(1) - - # Retrieve video playlist to extract media URL - # I'm not completely sure what all these options are, but we - # seem to need most of them, otherwise the server sends a 401. - yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents - yv_bitrate = '700' # according to Wikipedia this is hard-coded - request = compat_urllib_request.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id + - '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + - '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') - try: - self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to retrieve video webpage: %s' % compat_str(err)) - - # Extract media URL from playlist XML - mobj = re.search(r'.+?)";', webpage) + + if m_id is None: + # TODO: Check which url parameters are required + info_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;lmsoverride=1;outputformat=mrss;cb=974419660;id=%s;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id + webpage = self._download_webpage(info_url, video_id, u'Downloading info webpage') + info_re = r'''<!\[CDATA\[(?P<title>.*?)\]\]>.* + .*?)\]\]>.* + .*?)\ .*\]\]>.* + n: video_ids = video_ids[:n] videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids] - return videos + return self.playlist_result(videos, query) class GoogleSearchIE(InfoExtractor): """Information Extractor for Google Video search queries.""" _VALID_URL = r'gvsearch(?P|\d+|all):(?P[\s\S]+)' - _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"' + _MORE_PAGES_INDICATOR = r'id="pnnext" class="pn"' _max_google_results = 1000 IE_NAME = u'video.google:search' @@ -1490,7 +1439,7 @@ class GoogleSearchIE(InfoExtractor): } for pagenum in itertools.count(1): - result_url = u'http://video.google.com/videosearch?q=%s&start=%s&hl=en' % (compat_urllib_parse.quote_plus(query), pagenum*10) + result_url = u'http://www.google.com/search?tbm=vid&q=%s&start=%s&hl=en' % (compat_urllib_parse.quote_plus(query), pagenum*10) webpage = self._download_webpage(result_url, u'gvsearch:' + query, note='Downloading result page ' + str(pagenum)) @@ -1507,18 +1456,10 @@ class GoogleSearchIE(InfoExtractor): class YahooSearchIE(InfoExtractor): """Information Extractor for Yahoo! Video search queries.""" - _WORKING = False _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+' - _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s' - _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"' - _MORE_PAGES_INDICATOR = r'\s*Next' - _max_yahoo_results = 1000 - IE_NAME = u'video.yahoo:search' - def report_download_page(self, query, pagenum): - """Report attempt to download playlist page with given number.""" - query = query.decode(preferredencoding()) - self.to_screen(u'query "%s": Downloading page %s' % (query, pagenum)) + _max_yahoo_results = 1000 + IE_NAME = u'screen.yahoo:search' def _real_extract(self, query): mobj = re.match(self._VALID_URL, query) @@ -1529,11 +1470,9 @@ class YahooSearchIE(InfoExtractor): prefix = prefix[8:] query = query.encode('utf-8') if prefix == '': - self._download_n_results(query, 1) - return + return self._get_n_results(query, 1) elif prefix == 'all': - self._download_n_results(query, self._max_yahoo_results) - return + return self._get_n_results(query, self._max_yahoo_results) else: try: n = int(prefix) @@ -1542,46 +1481,36 @@ class YahooSearchIE(InfoExtractor): elif n > self._max_yahoo_results: self._downloader.report_warning(u'yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) n = self._max_yahoo_results - self._download_n_results(query, n) - return + return self._get_n_results(query, n) except ValueError: # parsing prefix as integer fails - self._download_n_results(query, 1) - return - - def _download_n_results(self, query, n): - """Downloads a specified number of results for a query""" - - video_ids = [] - already_seen = set() - pagenum = 1 + return self._get_n_results(query, 1) - while True: - self.report_download_page(query, pagenum) - result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum) - request = compat_urllib_request.Request(result_url) - try: - page = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download webpage: %s' % compat_str(err)) + def _get_n_results(self, query, n): + """Get a specified number of results for a query""" - # Extract video identifiers - for mobj in re.finditer(self._VIDEO_INDICATOR, page): - video_id = mobj.group(1) - if video_id not in already_seen: - video_ids.append(video_id) - already_seen.add(video_id) - if len(video_ids) == n: - # Specified n videos reached - for id in video_ids: - self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) - return - - if re.search(self._MORE_PAGES_INDICATOR, page) is None: - for id in video_ids: - self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) - return + res = { + '_type': 'playlist', + 'id': query, + 'entries': [] + } + for pagenum in itertools.count(0): + result_url = u'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30) + webpage = self._download_webpage(result_url, query, + note='Downloading results page '+str(pagenum+1)) + info = json.loads(webpage) + m = info[u'm'] + results = info[u'results'] + + for (i, r) in enumerate(results): + if (pagenum * 30) +i >= n: + break + mobj = re.search(r'(?Pscreen\.yahoo\.com/.*?-\d*?\.html)"', r) + e = self.url_result('http://' + mobj.group('url'), 'Yahoo') + res['entries'].append(e) + if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1 )): + break - pagenum = pagenum + 1 + return res class YoutubePlaylistIE(InfoExtractor): diff --git a/youtube_dl/PostProcessor.py b/youtube_dl/PostProcessor.py index 0c6885dda..8868b37af 100644 --- a/youtube_dl/PostProcessor.py +++ b/youtube_dl/PostProcessor.py @@ -85,8 +85,9 @@ class FFmpegPostProcessor(PostProcessor): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout,stderr = p.communicate() if p.returncode != 0: + stderr = stderr.decode('utf-8', 'replace') msg = stderr.strip().split('\n')[-1] - raise FFmpegPostProcessorError(msg.decode('utf-8', 'replace')) + raise FFmpegPostProcessorError(msg) def _ffmpeg_filename_argument(self, fn): # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 80f3b9f33..308c48fe6 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -191,8 +191,8 @@ def parseOpts(overrideArguments=None): action='store_true', dest='writesubtitles', help='write subtitle file (currently youtube only)', default=False) video_format.add_option('--only-sub', - action='store_true', dest='onlysubtitles', - help='downloads only the subtitles (no video)', default=False) + action='store_true', dest='skip_download', + help='[deprecated] alias of --skip-download', default=False) video_format.add_option('--all-subs', action='store_true', dest='allsubtitles', help='downloads all the available subtitles of the video (currently youtube only)', default=False) @@ -532,7 +532,6 @@ def _real_main(argv=None): 'writeinfojson': opts.writeinfojson, 'writethumbnail': opts.writethumbnail, 'writesubtitles': opts.writesubtitles, - 'onlysubtitles': opts.onlysubtitles, 'allsubtitles': opts.allsubtitles, 'listsubtitles': opts.listsubtitles, 'subtitlesformat': opts.subtitlesformat, diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index f2342b10a..616948e10 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import errno import gzip import io import json @@ -334,12 +335,20 @@ def sanitize_open(filename, open_mode): stream = open(encodeFilename(filename), open_mode) return (stream, filename) except (IOError, OSError) as err: - # In case of error, try to remove win32 forbidden chars - filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename) + if err.errno in (errno.EACCES,): + raise - # An exception here should be caught in the caller - stream = open(encodeFilename(filename), open_mode) - return (stream, filename) + # In case of error, try to remove win32 forbidden chars + alt_filename = os.path.join( + re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part) + for path_part in os.path.split(filename) + ) + if alt_filename == filename: + raise + else: + # An exception here should be caught in the caller + stream = open(encodeFilename(filename), open_mode) + return (stream, alt_filename) def timeconvert(timestr): diff --git a/youtube_dl/version.py b/youtube_dl/version.py index ae9688e17..6619cfecc 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.05.10' +__version__ = '2013.05.13'