2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # Author: Benjamin Johnson
6 # License: Public domain code
23 # parse_qs was moved from the cgi module to the urlparse module recently.
25 from urlparse import parse_qs
27 from cgi import parse_qs
30 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6',
31 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
32 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
33 'Accept-Language': 'en-us,en;q=0.5',
36 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
38 def preferredencoding():
39 """Get preferred encoding.
41 Returns the best encoding scheme for the system, based on
42 locale.getpreferredencoding() and some further tweaks.
44 def yield_preferredencoding():
46 pref = locale.getpreferredencoding()
52 return yield_preferredencoding().next()
54 def htmlentity_transform(matchobj):
55 """Transforms an HTML entity to a Unicode character.
57 This function receives a match object and is intended to be used with
58 the re.sub() function.
60 entity = matchobj.group(1)
62 # Known non-numeric HTML entity
63 if entity in htmlentitydefs.name2codepoint:
64 return unichr(htmlentitydefs.name2codepoint[entity])
67 mobj = re.match(ur'(?u)#(x?\d+)', entity)
69 numstr = mobj.group(1)
70 if numstr.startswith(u'x'):
72 numstr = u'0%s' % numstr
75 return unichr(long(numstr, base))
77 # Unknown entity in name, return its literal representation
78 return (u'&%s;' % entity)
80 def sanitize_title(utitle):
81 """Sanitizes a video title so it could be used as part of a filename."""
82 utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
83 return utitle.replace(unicode(os.sep), u'%')
85 def sanitize_open(filename, open_mode):
86 """Try to open the given filename, and slightly tweak it if this fails.
88 Attempts to open the given filename. If this fails, it tries to change
89 the filename slightly, step by step, until it's either able to open it
90 or it fails and raises a final exception, like the standard open()
93 It returns the tuple (stream, definitive_file_name).
97 return (sys.stdout, filename)
98 stream = open(filename, open_mode)
99 return (stream, filename)
100 except (IOError, OSError), err:
101 # In case of error, try to remove win32 forbidden chars
102 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
104 # An exception here should be caught in the caller
105 stream = open(filename, open_mode)
106 return (stream, filename)
109 class DownloadError(Exception):
110 """Download Error exception.
112 This exception may be thrown by FileDownloader objects if they are not
113 configured to continue on errors. They will contain the appropriate
118 class SameFileError(Exception):
119 """Same File exception.
121 This exception will be thrown by FileDownloader objects if they detect
122 multiple files would have to be downloaded to the same file on disk.
126 class PostProcessingError(Exception):
127 """Post Processing exception.
129 This exception may be raised by PostProcessor's .run() method to
130 indicate an error in the postprocessing task.
134 class UnavailableFormatError(Exception):
135 """Unavailable Format exception.
137 This exception will be thrown when a video is requested
138 in a format that is not available for that video.
142 class ContentTooShortError(Exception):
143 """Content Too Short exception.
145 This exception may be raised by FileDownloader objects when a file they
146 download is too small for what the server announced first, indicating
147 the connection was probably interrupted.
153 def __init__(self, downloaded, expected):
154 self.downloaded = downloaded
155 self.expected = expected
157 class FileDownloader(object):
158 """File Downloader class.
160 File downloader objects are the ones responsible of downloading the
161 actual video file and writing it to disk if the user has requested
162 it, among some other tasks. In most cases there should be one per
163 program. As, given a video URL, the downloader doesn't know how to
164 extract all the needed information, task that InfoExtractors do, it
165 has to pass the URL to one of them.
167 For this, file downloader objects have a method that allows
168 InfoExtractors to be registered in a given order. When it is passed
169 a URL, the file downloader handles it to the first InfoExtractor it
170 finds that reports being able to handle it. The InfoExtractor extracts
171 all the information about the video or videos the URL refers to, and
172 asks the FileDownloader to process the video information, possibly
173 downloading the video.
175 File downloaders accept a lot of parameters. In order not to saturate
176 the object constructor with arguments, it receives a dictionary of
177 options instead. These options are available through the params
178 attribute for the InfoExtractors to use. The FileDownloader also
179 registers itself as the downloader in charge for the InfoExtractors
180 that are added to it, so this is a "mutual registration".
184 username: Username for authentication purposes.
185 password: Password for authentication purposes.
186 usenetrc: Use netrc for authentication instead.
187 quiet: Do not print messages to stdout.
188 forceurl: Force printing final URL.
189 forcetitle: Force printing title.
190 simulate: Do not download the video files.
191 format: Video format code.
192 format_limit: Highest quality format to try.
193 outtmpl: Template for output names.
194 ignoreerrors: Do not stop on download errors.
195 ratelimit: Download speed limit, in bytes/sec.
196 nooverwrites: Prevent overwriting files.
197 retries: Number of times to retry for HTTP error 503
198 continuedl: Try to continue downloads if possible.
199 noprogress: Do not print the progress bar.
205 _download_retcode = None
206 _num_downloads = None
208 def __init__(self, params):
209 """Create a FileDownloader object with the given options."""
212 self._download_retcode = 0
213 self._num_downloads = 0
217 def pmkdir(filename):
218 """Create directory components in filename. Similar to Unix "mkdir -p"."""
219 components = filename.split(os.sep)
220 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
221 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
222 for dir in aggregate:
223 if not os.path.exists(dir):
227 def format_bytes(bytes):
230 if type(bytes) is str:
235 exponent = long(math.log(bytes, 1024.0))
236 suffix = 'bkMGTPEZY'[exponent]
237 converted = float(bytes) / float(1024**exponent)
238 return '%.2f%s' % (converted, suffix)
241 def calc_percent(byte_counter, data_len):
244 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
247 def calc_eta(start, now, total, current):
251 if current == 0 or dif < 0.001: # One millisecond
253 rate = float(current) / dif
254 eta = long((float(total) - float(current)) / rate)
255 (eta_mins, eta_secs) = divmod(eta, 60)
258 return '%02d:%02d' % (eta_mins, eta_secs)
261 def calc_speed(start, now, bytes):
263 if bytes == 0 or dif < 0.001: # One millisecond
264 return '%10s' % '---b/s'
265 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
268 def best_block_size(elapsed_time, bytes):
269 new_min = max(bytes / 2.0, 1.0)
270 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
271 if elapsed_time < 0.001:
273 rate = bytes / elapsed_time
281 def parse_bytes(bytestr):
282 """Parse a string indicating a byte quantity into a long integer."""
283 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
286 number = float(matchobj.group(1))
287 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
288 return long(round(number * multiplier))
292 """Verify a URL is valid and data could be downloaded. Return real data URL."""
293 request = urllib2.Request(url, None, std_headers)
294 data = urllib2.urlopen(request)
300 def add_info_extractor(self, ie):
301 """Add an InfoExtractor object to the end of the list."""
303 ie.set_downloader(self)
305 def add_post_processor(self, pp):
306 """Add a PostProcessor object to the end of the chain."""
308 pp.set_downloader(self)
310 def to_stdout(self, message, skip_eol=False, ignore_encoding_errors=False):
311 """Print message to stdout if not in quiet mode."""
313 if not self.params.get('quiet', False):
314 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
316 except (UnicodeEncodeError), err:
317 if not ignore_encoding_errors:
320 def to_stderr(self, message):
321 """Print message to stderr."""
322 print >>sys.stderr, message.encode(preferredencoding())
324 def fixed_template(self):
325 """Checks if the output template is fixed."""
326 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
328 def trouble(self, message=None):
329 """Determine action to take when a download problem appears.
331 Depending on if the downloader has been configured to ignore
332 download errors or not, this method may throw an exception or
333 not when errors are found, after printing the message.
335 if message is not None:
336 self.to_stderr(message)
337 if not self.params.get('ignoreerrors', False):
338 raise DownloadError(message)
339 self._download_retcode = 1
341 def slow_down(self, start_time, byte_counter):
342 """Sleep if the download speed is over the rate limit."""
343 rate_limit = self.params.get('ratelimit', None)
344 if rate_limit is None or byte_counter == 0:
347 elapsed = now - start_time
350 speed = float(byte_counter) / elapsed
351 if speed > rate_limit:
352 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
354 def report_destination(self, filename):
355 """Report destination filename."""
356 self.to_stdout(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
358 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
359 """Report download progress."""
360 if self.params.get('noprogress', False):
362 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
363 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
365 def report_resuming_byte(self, resume_len):
366 """Report attemtp to resume at given byte."""
367 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
369 def report_retry(self, count, retries):
370 """Report retry in case of HTTP error 503"""
371 self.to_stdout(u'[download] Got HTTP error 503. Retrying (attempt %d of %d)...' % (count, retries))
373 def report_file_already_downloaded(self, file_name):
374 """Report file has already been fully downloaded."""
376 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
377 except (UnicodeEncodeError), err:
378 self.to_stdout(u'[download] The file has already been downloaded')
380 def report_unable_to_resume(self):
381 """Report it was impossible to resume download."""
382 self.to_stdout(u'[download] Unable to resume')
384 def report_finish(self):
385 """Report download finished."""
386 if self.params.get('noprogress', False):
387 self.to_stdout(u'[download] Download completed')
391 def process_info(self, info_dict):
392 """Process a single dictionary returned by an InfoExtractor."""
393 # Do nothing else if in simulate mode
394 if self.params.get('simulate', False):
395 # Verify URL if it's an HTTP one
396 if info_dict['url'].startswith('http'):
398 self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8')
399 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
400 raise UnavailableFormatError
403 if self.params.get('forcetitle', False):
404 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
405 if self.params.get('forceurl', False):
406 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
407 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
408 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
409 if self.params.get('forcedescription', False) and 'description' in info_dict:
410 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
415 template_dict = dict(info_dict)
416 template_dict['epoch'] = unicode(long(time.time()))
417 template_dict['ord'] = unicode('%05d' % self._num_downloads)
418 filename = self.params['outtmpl'] % template_dict
419 except (ValueError, KeyError), err:
420 self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
421 if self.params.get('nooverwrites', False) and os.path.exists(filename):
422 self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
426 self.pmkdir(filename)
427 except (OSError, IOError), err:
428 self.trouble('ERROR: unable to create directories: %s' % str(err))
432 success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
433 except (OSError, IOError), err:
434 raise UnavailableFormatError
435 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
436 self.trouble('ERROR: unable to download video data: %s' % str(err))
438 except (ContentTooShortError, ), err:
439 self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
444 self.post_process(filename, info_dict)
445 except (PostProcessingError), err:
446 self.trouble('ERROR: postprocessing: %s' % str(err))
449 def download(self, url_list):
450 """Download a given list of URLs."""
451 if len(url_list) > 1 and self.fixed_template():
452 raise SameFileError(self.params['outtmpl'])
455 suitable_found = False
457 # Go to next InfoExtractor if not suitable
458 if not ie.suitable(url):
461 # Suitable InfoExtractor found
462 suitable_found = True
464 # Extract information from URL and process it
467 # Suitable InfoExtractor had been found; go to next URL
470 if not suitable_found:
471 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
473 return self._download_retcode
475 def post_process(self, filename, ie_info):
476 """Run the postprocessing chain on the given file."""
478 info['filepath'] = filename
484 def _download_with_rtmpdump(self, filename, url, player_url):
485 self.report_destination(filename)
487 # Check for rtmpdump first
489 subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
490 except (OSError, IOError):
491 self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
494 # Download using rtmpdump. rtmpdump returns exit code 2 when
495 # the connection was interrumpted and resuming appears to be
496 # possible. This is part of rtmpdump's normal usage, AFAIK.
497 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename]
498 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
499 while retval == 2 or retval == 1:
500 prevsize = os.path.getsize(filename)
501 self.to_stdout(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
502 time.sleep(5.0) # This seems to be needed
503 retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
504 cursize = os.path.getsize(filename)
505 if prevsize == cursize and retval == 1:
508 self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
511 self.trouble('\nERROR: rtmpdump exited with code %d' % retval)
514 def _do_download(self, filename, url, player_url):
515 # Attempt to download using rtmpdump
516 if url.startswith('rtmp'):
517 return self._download_with_rtmpdump(filename, url, player_url)
521 basic_request = urllib2.Request(url, None, std_headers)
522 request = urllib2.Request(url, None, std_headers)
524 # Establish possible resume length
525 if os.path.isfile(filename):
526 resume_len = os.path.getsize(filename)
530 # Request parameters in case of being able to resume
531 if self.params.get('continuedl', False) and resume_len != 0:
532 self.report_resuming_byte(resume_len)
533 request.add_header('Range','bytes=%d-' % resume_len)
537 retries = self.params.get('retries', 0)
539 # Establish connection
541 data = urllib2.urlopen(request)
543 except (urllib2.HTTPError, ), err:
545 # Retry in case of HTTP error 503
548 self.report_retry(count, retries)
550 if err.code != 416: # 416 is 'Requested range not satisfiable'
553 data = urllib2.urlopen(basic_request)
554 content_length = data.info()['Content-Length']
556 if content_length is not None and long(content_length) == resume_len:
557 # Because the file had already been fully downloaded
558 self.report_file_already_downloaded(filename)
561 # Because the server didn't let us
562 self.report_unable_to_resume()
565 data_len = data.info().get('Content-length', None)
566 data_len_str = self.format_bytes(data_len)
573 data_block = data.read(block_size)
575 data_block_len = len(data_block)
576 if data_block_len == 0:
578 byte_counter += data_block_len
580 # Open file just in time
583 (stream, filename) = sanitize_open(filename, open_mode)
584 self.report_destination(filename)
585 self._num_downloads += 1
586 except (OSError, IOError), err:
587 self.trouble('ERROR: unable to open for writing: %s' % str(err))
590 stream.write(data_block)
591 except (IOError, OSError), err:
592 self.trouble('\nERROR: unable to write data: %s' % str(err))
593 block_size = self.best_block_size(after - before, data_block_len)
596 percent_str = self.calc_percent(byte_counter, data_len)
597 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
598 speed_str = self.calc_speed(start, time.time(), byte_counter)
599 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
602 self.slow_down(start, byte_counter)
605 if data_len is not None and str(byte_counter) != data_len:
606 raise ContentTooShortError(byte_counter, long(data_len))
609 class InfoExtractor(object):
610 """Information Extractor class.
612 Information extractors are the classes that, given a URL, extract
613 information from the video (or videos) the URL refers to. This
614 information includes the real video URL, the video title and simplified
615 title, author and others. The information is stored in a dictionary
616 which is then passed to the FileDownloader. The FileDownloader
617 processes this information possibly downloading the video to the file
618 system, among other possible outcomes. The dictionaries must include
619 the following fields:
621 id: Video identifier.
622 url: Final video URL.
623 uploader: Nickname of the video uploader.
624 title: Literal title.
625 stitle: Simplified title.
626 ext: Video filename extension.
627 format: Video format.
628 player_url: SWF Player URL (may be None).
630 The following fields are optional. Their primary purpose is to allow
631 youtube-dl to serve as the backend for a video search function, such
632 as the one in youtube2mp3. They are only used when their respective
633 forced printing functions are called:
635 thumbnail: Full URL to a video thumbnail image.
636 description: One-line video description.
638 Subclasses of this one should re-define the _real_initialize() and
639 _real_extract() methods, as well as the suitable() static method.
640 Probably, they should also be instantiated and added to the main
647 def __init__(self, downloader=None):
648 """Constructor. Receives an optional downloader."""
650 self.set_downloader(downloader)
654 """Receives a URL and returns True if suitable for this IE."""
657 def initialize(self):
658 """Initializes an instance (authentication, etc)."""
660 self._real_initialize()
663 def extract(self, url):
664 """Extracts URL information and returns it in list of dicts."""
666 return self._real_extract(url)
668 def set_downloader(self, downloader):
669 """Sets the downloader for this IE."""
670 self._downloader = downloader
672 def _real_initialize(self):
673 """Real initialization process. Redefine in subclasses."""
676 def _real_extract(self, url):
677 """Real extraction process. Redefine in subclasses."""
680 class YoutubeIE(InfoExtractor):
681 """Information extractor for youtube.com."""
683 _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?[\?#](?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
684 _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
685 _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
686 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
687 _NETRC_MACHINE = 'youtube'
688 # Listed in order of priority for the -b option
689 _available_formats = ['38', '37', '22', '45', '35', '34', '43', '18', '6', '5', '17', '13', None]
690 _video_extensions = {
703 return (re.match(YoutubeIE._VALID_URL, url) is not None)
705 def report_lang(self):
706 """Report attempt to set language."""
707 self._downloader.to_stdout(u'[youtube] Setting language')
709 def report_login(self):
710 """Report attempt to log in."""
711 self._downloader.to_stdout(u'[youtube] Logging in')
713 def report_age_confirmation(self):
714 """Report attempt to confirm age."""
715 self._downloader.to_stdout(u'[youtube] Confirming age')
717 def report_video_webpage_download(self, video_id):
718 """Report attempt to download video webpage."""
719 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
721 def report_video_info_webpage_download(self, video_id):
722 """Report attempt to download video info webpage."""
723 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
725 def report_information_extraction(self, video_id):
726 """Report attempt to extract video information."""
727 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
729 def report_unavailable_format(self, video_id, format):
730 """Report extracted video URL."""
731 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
733 def report_rtmp_download(self):
734 """Indicate the download will use the RTMP protocol."""
735 self._downloader.to_stdout(u'[youtube] RTMP download detected')
737 def _real_initialize(self):
738 if self._downloader is None:
743 downloader_params = self._downloader.params
745 # Attempt to use provided username and password or .netrc data
746 if downloader_params.get('username', None) is not None:
747 username = downloader_params['username']
748 password = downloader_params['password']
749 elif downloader_params.get('usenetrc', False):
751 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
756 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
757 except (IOError, netrc.NetrcParseError), err:
758 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
762 request = urllib2.Request(self._LANG_URL, None, std_headers)
765 urllib2.urlopen(request).read()
766 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
767 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
770 # No authentication to be performed
776 'current_form': 'loginForm',
778 'action_login': 'Log In',
779 'username': username,
780 'password': password,
782 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
785 login_results = urllib2.urlopen(request).read()
786 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
787 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
789 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
790 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
796 'action_confirm': 'Confirm',
798 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
800 self.report_age_confirmation()
801 age_results = urllib2.urlopen(request).read()
802 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
803 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
806 def _real_extract(self, url):
807 # Extract video id from URL
808 mobj = re.match(self._VALID_URL, url)
810 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
812 video_id = mobj.group(2)
814 # Downloader parameters
819 if self._downloader is not None:
820 params = self._downloader.params
821 format_param = params.get('format', None)
822 if format_param == '0':
823 format_limit = params.get('format_limit', None)
824 if format_limit is not None:
826 # Start at a different format if the user has limited the maximum quality
827 quality_index = self._available_formats.index(format_limit)
830 format_param = self._available_formats[quality_index]
832 elif format_param == '-1':
833 format_param = self._available_formats[quality_index]
838 video_extension = self._video_extensions.get(format_param, 'flv')
841 self.report_video_webpage_download(video_id)
842 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id, None, std_headers)
844 video_webpage = urllib2.urlopen(request).read()
845 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
846 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
849 # Attempt to extract SWF player URL
850 mobj = re.search(r'swfConfig.*"(http://.*?watch.*?-.*?\.swf)"', video_webpage)
852 player_url = mobj.group(1)
857 self.report_video_info_webpage_download(video_id)
858 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
859 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
860 % (video_id, el_type))
861 request = urllib2.Request(video_info_url, None, std_headers)
863 video_info_webpage = urllib2.urlopen(request).read()
864 video_info = parse_qs(video_info_webpage)
865 if 'token' in video_info:
867 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
868 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
870 self.report_information_extraction(video_id)
873 if 'token' not in video_info:
874 # Attempt to see if YouTube has issued an error message
875 if 'reason' not in video_info:
876 self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
877 stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
878 stream.write(video_info_webpage)
881 reason = urllib.unquote_plus(video_info['reason'][0])
882 self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
884 token = urllib.unquote_plus(video_info['token'][0])
885 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
886 if format_param is not None:
887 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
889 # Check possible RTMP download
890 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
891 self.report_rtmp_download()
892 video_real_url = video_info['conn'][0]
895 if 'author' not in video_info:
896 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
898 video_uploader = urllib.unquote_plus(video_info['author'][0])
901 if 'title' not in video_info:
902 self._downloader.trouble(u'ERROR: unable to extract video title')
904 video_title = urllib.unquote_plus(video_info['title'][0])
905 video_title = video_title.decode('utf-8')
906 video_title = sanitize_title(video_title)
909 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
910 simple_title = simple_title.strip(ur'_')
913 if 'thumbnail_url' not in video_info:
914 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
916 else: # don't panic if we can't find it
917 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
920 video_description = 'No description available.'
921 if self._downloader.params.get('forcedescription', False):
922 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
924 video_description = mobj.group(1)
927 # Process video information
928 self._downloader.process_info({
929 'id': video_id.decode('utf-8'),
930 'url': video_real_url.decode('utf-8'),
931 'uploader': video_uploader.decode('utf-8'),
932 'title': video_title,
933 'stitle': simple_title,
934 'ext': video_extension.decode('utf-8'),
935 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
936 'thumbnail': video_thumbnail.decode('utf-8'),
937 'description': video_description.decode('utf-8'),
938 'player_url': player_url,
943 if quality_index == len(self._available_formats):
947 format_param = self._available_formats[quality_index]
951 except UnavailableFormatError, err:
952 if best_quality or all_formats:
954 if quality_index == len(self._available_formats):
955 # I don't ever expect this to happen
957 self._downloader.trouble(u'ERROR: no known formats available for video')
960 self.report_unavailable_format(video_id, format_param)
961 format_param = self._available_formats[quality_index]
964 self._downloader.trouble('ERROR: format not available for video')
968 class MetacafeIE(InfoExtractor):
969 """Information Extractor for metacafe.com."""
971 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
972 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
973 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
976 def __init__(self, youtube_ie, downloader=None):
977 InfoExtractor.__init__(self, downloader)
978 self._youtube_ie = youtube_ie
982 return (re.match(MetacafeIE._VALID_URL, url) is not None)
984 def report_disclaimer(self):
985 """Report disclaimer retrieval."""
986 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
988 def report_age_confirmation(self):
989 """Report attempt to confirm age."""
990 self._downloader.to_stdout(u'[metacafe] Confirming age')
992 def report_download_webpage(self, video_id):
993 """Report webpage download."""
994 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
996 def report_extraction(self, video_id):
997 """Report information extraction."""
998 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
1000 def _real_initialize(self):
1001 # Retrieve disclaimer
1002 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
1004 self.report_disclaimer()
1005 disclaimer = urllib2.urlopen(request).read()
1006 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1007 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1013 'submit': "Continue - I'm over 18",
1015 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
1017 self.report_age_confirmation()
1018 disclaimer = urllib2.urlopen(request).read()
1019 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1020 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1023 def _real_extract(self, url):
1024 # Extract id and simplified title from URL
1025 mobj = re.match(self._VALID_URL, url)
1027 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1030 video_id = mobj.group(1)
1032 # Check if video comes from YouTube
1033 mobj2 = re.match(r'^yt-(.*)$', video_id)
1034 if mobj2 is not None:
1035 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1038 simple_title = mobj.group(2).decode('utf-8')
1039 video_extension = 'flv'
1041 # Retrieve video webpage to extract further information
1042 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1044 self.report_download_webpage(video_id)
1045 webpage = urllib2.urlopen(request).read()
1046 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1047 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1050 # Extract URL, uploader and title from webpage
1051 self.report_extraction(video_id)
1052 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1054 self._downloader.trouble(u'ERROR: unable to extract media URL')
1056 mediaURL = urllib.unquote(mobj.group(1))
1058 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1060 # self._downloader.trouble(u'ERROR: unable to extract gdaKey')
1062 #gdaKey = mobj.group(1)
1064 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1066 video_url = mediaURL
1068 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1070 self._downloader.trouble(u'ERROR: unable to extract title')
1072 video_title = mobj.group(1).decode('utf-8')
1073 video_title = sanitize_title(video_title)
1075 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1077 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1079 video_uploader = mobj.group(1)
1082 # Process video information
1083 self._downloader.process_info({
1084 'id': video_id.decode('utf-8'),
1085 'url': video_url.decode('utf-8'),
1086 'uploader': video_uploader.decode('utf-8'),
1087 'title': video_title,
1088 'stitle': simple_title,
1089 'ext': video_extension.decode('utf-8'),
1093 except UnavailableFormatError:
1094 self._downloader.trouble(u'ERROR: format not available for video')
1097 class DailymotionIE(InfoExtractor):
1098 """Information Extractor for Dailymotion"""
1100 _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1102 def __init__(self, downloader=None):
1103 InfoExtractor.__init__(self, downloader)
1107 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1109 def report_download_webpage(self, video_id):
1110 """Report webpage download."""
1111 self._downloader.to_stdout(u'[dailymotion] %s: Downloading webpage' % video_id)
1113 def report_extraction(self, video_id):
1114 """Report information extraction."""
1115 self._downloader.to_stdout(u'[dailymotion] %s: Extracting information' % video_id)
1117 def _real_initialize(self):
1120 def _real_extract(self, url):
1121 # Extract id and simplified title from URL
1122 mobj = re.match(self._VALID_URL, url)
1124 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1127 video_id = mobj.group(1)
1129 simple_title = mobj.group(2).decode('utf-8')
1130 video_extension = 'flv'
1132 # Retrieve video webpage to extract further information
1133 request = urllib2.Request(url)
1135 self.report_download_webpage(video_id)
1136 webpage = urllib2.urlopen(request).read()
1137 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1138 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1141 # Extract URL, uploader and title from webpage
1142 self.report_extraction(video_id)
1143 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1145 self._downloader.trouble(u'ERROR: unable to extract media URL')
1147 mediaURL = urllib.unquote(mobj.group(1))
1149 # if needed add http://www.dailymotion.com/ if relative URL
1151 video_url = mediaURL
1153 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1154 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1156 self._downloader.trouble(u'ERROR: unable to extract title')
1158 video_title = mobj.group(1).decode('utf-8')
1159 video_title = sanitize_title(video_title)
1161 mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a></div>', webpage)
1163 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1165 video_uploader = mobj.group(1)
1168 # Process video information
1169 self._downloader.process_info({
1170 'id': video_id.decode('utf-8'),
1171 'url': video_url.decode('utf-8'),
1172 'uploader': video_uploader.decode('utf-8'),
1173 'title': video_title,
1174 'stitle': simple_title,
1175 'ext': video_extension.decode('utf-8'),
1179 except UnavailableFormatError:
1180 self._downloader.trouble(u'ERROR: format not available for video')
1182 class GoogleIE(InfoExtractor):
1183 """Information extractor for video.google.com."""
1185 _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1187 def __init__(self, downloader=None):
1188 InfoExtractor.__init__(self, downloader)
1192 return (re.match(GoogleIE._VALID_URL, url) is not None)
1194 def report_download_webpage(self, video_id):
1195 """Report webpage download."""
1196 self._downloader.to_stdout(u'[video.google] %s: Downloading webpage' % video_id)
1198 def report_extraction(self, video_id):
1199 """Report information extraction."""
1200 self._downloader.to_stdout(u'[video.google] %s: Extracting information' % video_id)
1202 def _real_initialize(self):
1205 def _real_extract(self, url):
1206 # Extract id from URL
1207 mobj = re.match(self._VALID_URL, url)
1209 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1212 video_id = mobj.group(1)
1214 video_extension = 'mp4'
1216 # Retrieve video webpage to extract further information
1217 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1219 self.report_download_webpage(video_id)
1220 webpage = urllib2.urlopen(request).read()
1221 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1222 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1225 # Extract URL, uploader, and title from webpage
1226 self.report_extraction(video_id)
1227 mobj = re.search(r"download_url:'([^']+)'", webpage)
1229 video_extension = 'flv'
1230 mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1232 self._downloader.trouble(u'ERROR: unable to extract media URL')
1234 mediaURL = urllib.unquote(mobj.group(1))
1235 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1236 mediaURL = mediaURL.replace('\\x26', '\x26')
1238 video_url = mediaURL
1240 mobj = re.search(r'<title>(.*)</title>', webpage)
1242 self._downloader.trouble(u'ERROR: unable to extract title')
1244 video_title = mobj.group(1).decode('utf-8')
1245 video_title = sanitize_title(video_title)
1246 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1248 # Extract video description
1249 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1251 self._downloader.trouble(u'ERROR: unable to extract video description')
1253 video_description = mobj.group(1).decode('utf-8')
1254 if not video_description:
1255 video_description = 'No description available.'
1257 # Extract video thumbnail
1258 if self._downloader.params.get('forcethumbnail', False):
1259 request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1261 webpage = urllib2.urlopen(request).read()
1262 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1263 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1265 mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1267 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1269 video_thumbnail = mobj.group(1)
1270 else: # we need something to pass to process_info
1271 video_thumbnail = ''
1275 # Process video information
1276 self._downloader.process_info({
1277 'id': video_id.decode('utf-8'),
1278 'url': video_url.decode('utf-8'),
1280 'title': video_title,
1281 'stitle': simple_title,
1282 'ext': video_extension.decode('utf-8'),
1286 except UnavailableFormatError:
1287 self._downloader.trouble(u'ERROR: format not available for video')
1290 class PhotobucketIE(InfoExtractor):
1291 """Information extractor for photobucket.com."""
1293 _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1295 def __init__(self, downloader=None):
1296 InfoExtractor.__init__(self, downloader)
1300 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1302 def report_download_webpage(self, video_id):
1303 """Report webpage download."""
1304 self._downloader.to_stdout(u'[photobucket] %s: Downloading webpage' % video_id)
1306 def report_extraction(self, video_id):
1307 """Report information extraction."""
1308 self._downloader.to_stdout(u'[photobucket] %s: Extracting information' % video_id)
1310 def _real_initialize(self):
1313 def _real_extract(self, url):
1314 # Extract id from URL
1315 mobj = re.match(self._VALID_URL, url)
1317 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1320 video_id = mobj.group(1)
1322 video_extension = 'flv'
1324 # Retrieve video webpage to extract further information
1325 request = urllib2.Request(url)
1327 self.report_download_webpage(video_id)
1328 webpage = urllib2.urlopen(request).read()
1329 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1330 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1333 # Extract URL, uploader, and title from webpage
1334 self.report_extraction(video_id)
1335 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1337 self._downloader.trouble(u'ERROR: unable to extract media URL')
1339 mediaURL = urllib.unquote(mobj.group(1))
1341 video_url = mediaURL
1343 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1345 self._downloader.trouble(u'ERROR: unable to extract title')
1347 video_title = mobj.group(1).decode('utf-8')
1348 video_title = sanitize_title(video_title)
1349 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1351 video_uploader = mobj.group(2).decode('utf-8')
1354 # Process video information
1355 self._downloader.process_info({
1356 'id': video_id.decode('utf-8'),
1357 'url': video_url.decode('utf-8'),
1358 'uploader': video_uploader,
1359 'title': video_title,
1360 'stitle': simple_title,
1361 'ext': video_extension.decode('utf-8'),
1365 except UnavailableFormatError:
1366 self._downloader.trouble(u'ERROR: format not available for video')
1369 class YahooIE(InfoExtractor):
1370 """Information extractor for video.yahoo.com."""
1372 # _VALID_URL matches all Yahoo! Video URLs
1373 # _VPAGE_URL matches only the extractable '/watch/' URLs
1374 _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1375 _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1377 def __init__(self, downloader=None):
1378 InfoExtractor.__init__(self, downloader)
1382 return (re.match(YahooIE._VALID_URL, url) is not None)
1384 def report_download_webpage(self, video_id):
1385 """Report webpage download."""
1386 self._downloader.to_stdout(u'[video.yahoo] %s: Downloading webpage' % video_id)
1388 def report_extraction(self, video_id):
1389 """Report information extraction."""
1390 self._downloader.to_stdout(u'[video.yahoo] %s: Extracting information' % video_id)
1392 def _real_initialize(self):
1395 def _real_extract(self, url):
1396 # Extract ID from URL
1397 mobj = re.match(self._VALID_URL, url)
1399 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1402 video_id = mobj.group(2)
1403 video_extension = 'flv'
1405 # Rewrite valid but non-extractable URLs as
1406 # extractable English language /watch/ URLs
1407 if re.match(self._VPAGE_URL, url) is None:
1408 request = urllib2.Request(url)
1410 webpage = urllib2.urlopen(request).read()
1411 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1412 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1415 mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1417 self._downloader.trouble(u'ERROR: Unable to extract id field')
1419 yahoo_id = mobj.group(1)
1421 mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1423 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1425 yahoo_vid = mobj.group(1)
1427 url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1428 return self._real_extract(url)
1430 # Retrieve video webpage to extract further information
1431 request = urllib2.Request(url)
1433 self.report_download_webpage(video_id)
1434 webpage = urllib2.urlopen(request).read()
1435 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1436 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1439 # Extract uploader and title from webpage
1440 self.report_extraction(video_id)
1441 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1443 self._downloader.trouble(u'ERROR: unable to extract video title')
1445 video_title = mobj.group(1).decode('utf-8')
1446 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1448 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1450 self._downloader.trouble(u'ERROR: unable to extract video uploader')
1452 video_uploader = mobj.group(1).decode('utf-8')
1454 # Extract video thumbnail
1455 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1457 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1459 video_thumbnail = mobj.group(1).decode('utf-8')
1461 # Extract video description
1462 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1464 self._downloader.trouble(u'ERROR: unable to extract video description')
1466 video_description = mobj.group(1).decode('utf-8')
1467 if not video_description: video_description = 'No description available.'
1469 # Extract video height and width
1470 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1472 self._downloader.trouble(u'ERROR: unable to extract video height')
1474 yv_video_height = mobj.group(1)
1476 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1478 self._downloader.trouble(u'ERROR: unable to extract video width')
1480 yv_video_width = mobj.group(1)
1482 # Retrieve video playlist to extract media URL
1483 # I'm not completely sure what all these options are, but we
1484 # seem to need most of them, otherwise the server sends a 401.
1485 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
1486 yv_bitrate = '700' # according to Wikipedia this is hard-coded
1487 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1488 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1489 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1491 self.report_download_webpage(video_id)
1492 webpage = urllib2.urlopen(request).read()
1493 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1494 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1497 # Extract media URL from playlist XML
1498 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1500 self._downloader.trouble(u'ERROR: Unable to extract media URL')
1502 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1503 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1506 # Process video information
1507 self._downloader.process_info({
1508 'id': video_id.decode('utf-8'),
1510 'uploader': video_uploader,
1511 'title': video_title,
1512 'stitle': simple_title,
1513 'ext': video_extension.decode('utf-8'),
1514 'thumbnail': video_thumbnail.decode('utf-8'),
1515 'description': video_description,
1516 'thumbnail': video_thumbnail,
1517 'description': video_description,
1520 except UnavailableFormatError:
1521 self._downloader.trouble(u'ERROR: format not available for video')
1524 class GenericIE(InfoExtractor):
1525 """Generic last-resort information extractor."""
1527 def __init__(self, downloader=None):
1528 InfoExtractor.__init__(self, downloader)
1534 def report_download_webpage(self, video_id):
1535 """Report webpage download."""
1536 self._downloader.to_stdout(u'WARNING: Falling back on generic information extractor.')
1537 self._downloader.to_stdout(u'[generic] %s: Downloading webpage' % video_id)
1539 def report_extraction(self, video_id):
1540 """Report information extraction."""
1541 self._downloader.to_stdout(u'[generic] %s: Extracting information' % video_id)
1543 def _real_initialize(self):
1546 def _real_extract(self, url):
1547 video_id = url.split('/')[-1]
1548 request = urllib2.Request(url)
1550 self.report_download_webpage(video_id)
1551 webpage = urllib2.urlopen(request).read()
1552 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1553 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1555 except ValueError, err:
1556 # since this is the last-resort InfoExtractor, if
1557 # this error is thrown, it'll be thrown here
1558 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1561 # Start with something easy: JW Player in SWFObject
1562 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1564 # Broaden the search a little bit
1565 mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1567 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1570 # It's possible that one of the regexes
1571 # matched, but returned an empty group:
1572 if mobj.group(1) is None:
1573 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1576 video_url = urllib.unquote(mobj.group(1))
1577 video_id = os.path.basename(video_url)
1579 # here's a fun little line of code for you:
1580 video_extension = os.path.splitext(video_id)[1][1:]
1581 video_id = os.path.splitext(video_id)[0]
1583 # it's tempting to parse this further, but you would
1584 # have to take into account all the variations like
1585 # Video Title - Site Name
1586 # Site Name | Video Title
1587 # Video Title - Tagline | Site Name
1588 # and so on and so forth; it's just not practical
1589 mobj = re.search(r'<title>(.*)</title>', webpage)
1591 self._downloader.trouble(u'ERROR: unable to extract title')
1593 video_title = mobj.group(1).decode('utf-8')
1594 video_title = sanitize_title(video_title)
1595 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1597 # video uploader is domain name
1598 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1600 self._downloader.trouble(u'ERROR: unable to extract title')
1602 video_uploader = mobj.group(1).decode('utf-8')
1605 # Process video information
1606 self._downloader.process_info({
1607 'id': video_id.decode('utf-8'),
1608 'url': video_url.decode('utf-8'),
1609 'uploader': video_uploader,
1610 'title': video_title,
1611 'stitle': simple_title,
1612 'ext': video_extension.decode('utf-8'),
1616 except UnavailableFormatError:
1617 self._downloader.trouble(u'ERROR: format not available for video')
1620 class YoutubeSearchIE(InfoExtractor):
1621 """Information Extractor for YouTube search queries."""
1622 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1623 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1624 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1625 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1627 _max_youtube_results = 1000
1629 def __init__(self, youtube_ie, downloader=None):
1630 InfoExtractor.__init__(self, downloader)
1631 self._youtube_ie = youtube_ie
1635 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1637 def report_download_page(self, query, pagenum):
1638 """Report attempt to download playlist page with given number."""
1639 query = query.decode(preferredencoding())
1640 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1642 def _real_initialize(self):
1643 self._youtube_ie.initialize()
1645 def _real_extract(self, query):
1646 mobj = re.match(self._VALID_QUERY, query)
1648 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1651 prefix, query = query.split(':')
1653 query = query.encode('utf-8')
1655 self._download_n_results(query, 1)
1657 elif prefix == 'all':
1658 self._download_n_results(query, self._max_youtube_results)
1664 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1666 elif n > self._max_youtube_results:
1667 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1668 n = self._max_youtube_results
1669 self._download_n_results(query, n)
1671 except ValueError: # parsing prefix as integer fails
1672 self._download_n_results(query, 1)
1675 def _download_n_results(self, query, n):
1676 """Downloads a specified number of results for a query"""
1679 already_seen = set()
1683 self.report_download_page(query, pagenum)
1684 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1685 request = urllib2.Request(result_url, None, std_headers)
1687 page = urllib2.urlopen(request).read()
1688 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1689 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1692 # Extract video identifiers
1693 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1694 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1695 if video_id not in already_seen:
1696 video_ids.append(video_id)
1697 already_seen.add(video_id)
1698 if len(video_ids) == n:
1699 # Specified n videos reached
1700 for id in video_ids:
1701 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1704 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1705 for id in video_ids:
1706 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1709 pagenum = pagenum + 1
1711 class GoogleSearchIE(InfoExtractor):
1712 """Information Extractor for Google Video search queries."""
1713 _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
1714 _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1715 _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
1716 _MORE_PAGES_INDICATOR = r'<span>Next</span>'
1718 _max_google_results = 1000
1720 def __init__(self, google_ie, downloader=None):
1721 InfoExtractor.__init__(self, downloader)
1722 self._google_ie = google_ie
1726 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
1728 def report_download_page(self, query, pagenum):
1729 """Report attempt to download playlist page with given number."""
1730 query = query.decode(preferredencoding())
1731 self._downloader.to_stdout(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1733 def _real_initialize(self):
1734 self._google_ie.initialize()
1736 def _real_extract(self, query):
1737 mobj = re.match(self._VALID_QUERY, query)
1739 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1742 prefix, query = query.split(':')
1744 query = query.encode('utf-8')
1746 self._download_n_results(query, 1)
1748 elif prefix == 'all':
1749 self._download_n_results(query, self._max_google_results)
1755 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1757 elif n > self._max_google_results:
1758 self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1759 n = self._max_google_results
1760 self._download_n_results(query, n)
1762 except ValueError: # parsing prefix as integer fails
1763 self._download_n_results(query, 1)
1766 def _download_n_results(self, query, n):
1767 """Downloads a specified number of results for a query"""
1770 already_seen = set()
1774 self.report_download_page(query, pagenum)
1775 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1776 request = urllib2.Request(result_url, None, std_headers)
1778 page = urllib2.urlopen(request).read()
1779 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1780 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1783 # Extract video identifiers
1784 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1785 video_id = mobj.group(1)
1786 if video_id not in already_seen:
1787 video_ids.append(video_id)
1788 already_seen.add(video_id)
1789 if len(video_ids) == n:
1790 # Specified n videos reached
1791 for id in video_ids:
1792 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1795 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1796 for id in video_ids:
1797 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1800 pagenum = pagenum + 1
1802 class YahooSearchIE(InfoExtractor):
1803 """Information Extractor for Yahoo! Video search queries."""
1804 _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
1805 _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1806 _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1807 _MORE_PAGES_INDICATOR = r'\s*Next'
1809 _max_yahoo_results = 1000
1811 def __init__(self, yahoo_ie, downloader=None):
1812 InfoExtractor.__init__(self, downloader)
1813 self._yahoo_ie = yahoo_ie
1817 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
1819 def report_download_page(self, query, pagenum):
1820 """Report attempt to download playlist page with given number."""
1821 query = query.decode(preferredencoding())
1822 self._downloader.to_stdout(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1824 def _real_initialize(self):
1825 self._yahoo_ie.initialize()
1827 def _real_extract(self, query):
1828 mobj = re.match(self._VALID_QUERY, query)
1830 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1833 prefix, query = query.split(':')
1835 query = query.encode('utf-8')
1837 self._download_n_results(query, 1)
1839 elif prefix == 'all':
1840 self._download_n_results(query, self._max_yahoo_results)
1846 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1848 elif n > self._max_yahoo_results:
1849 self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1850 n = self._max_yahoo_results
1851 self._download_n_results(query, n)
1853 except ValueError: # parsing prefix as integer fails
1854 self._download_n_results(query, 1)
1857 def _download_n_results(self, query, n):
1858 """Downloads a specified number of results for a query"""
1861 already_seen = set()
1865 self.report_download_page(query, pagenum)
1866 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1867 request = urllib2.Request(result_url, None, std_headers)
1869 page = urllib2.urlopen(request).read()
1870 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1871 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1874 # Extract video identifiers
1875 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1876 video_id = mobj.group(1)
1877 if video_id not in already_seen:
1878 video_ids.append(video_id)
1879 already_seen.add(video_id)
1880 if len(video_ids) == n:
1881 # Specified n videos reached
1882 for id in video_ids:
1883 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1886 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1887 for id in video_ids:
1888 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1891 pagenum = pagenum + 1
1893 class YoutubePlaylistIE(InfoExtractor):
1894 """Information Extractor for YouTube playlists."""
1896 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
1897 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1898 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1899 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1902 def __init__(self, youtube_ie, downloader=None):
1903 InfoExtractor.__init__(self, downloader)
1904 self._youtube_ie = youtube_ie
1908 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1910 def report_download_page(self, playlist_id, pagenum):
1911 """Report attempt to download playlist page with given number."""
1912 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1914 def _real_initialize(self):
1915 self._youtube_ie.initialize()
1917 def _real_extract(self, url):
1918 # Extract playlist id
1919 mobj = re.match(self._VALID_URL, url)
1921 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1924 # Download playlist pages
1925 playlist_id = mobj.group(1)
1930 self.report_download_page(playlist_id, pagenum)
1931 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1933 page = urllib2.urlopen(request).read()
1934 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1935 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1938 # Extract video identifiers
1940 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1941 if mobj.group(1) not in ids_in_page:
1942 ids_in_page.append(mobj.group(1))
1943 video_ids.extend(ids_in_page)
1945 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1947 pagenum = pagenum + 1
1949 for id in video_ids:
1950 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1953 class YoutubeUserIE(InfoExtractor):
1954 """Information Extractor for YouTube users."""
1956 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
1957 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1958 _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
1961 def __init__(self, youtube_ie, downloader=None):
1962 InfoExtractor.__init__(self, downloader)
1963 self._youtube_ie = youtube_ie
1967 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
1969 def report_download_page(self, username):
1970 """Report attempt to download user page."""
1971 self._downloader.to_stdout(u'[youtube] user %s: Downloading page ' % (username))
1973 def _real_initialize(self):
1974 self._youtube_ie.initialize()
1976 def _real_extract(self, url):
1978 mobj = re.match(self._VALID_URL, url)
1980 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1983 # Download user page
1984 username = mobj.group(1)
1988 self.report_download_page(username)
1989 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
1991 page = urllib2.urlopen(request).read()
1992 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1993 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1996 # Extract video identifiers
1999 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2000 if mobj.group(1) not in ids_in_page:
2001 ids_in_page.append(mobj.group(1))
2002 video_ids.extend(ids_in_page)
2004 for id in video_ids:
2005 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2008 class PostProcessor(object):
2009 """Post Processor class.
2011 PostProcessor objects can be added to downloaders with their
2012 add_post_processor() method. When the downloader has finished a
2013 successful download, it will take its internal chain of PostProcessors
2014 and start calling the run() method on each one of them, first with
2015 an initial argument and then with the returned value of the previous
2018 The chain will be stopped if one of them ever returns None or the end
2019 of the chain is reached.
2021 PostProcessor objects follow a "mutual registration" process similar
2022 to InfoExtractor objects.
2027 def __init__(self, downloader=None):
2028 self._downloader = downloader
2030 def set_downloader(self, downloader):
2031 """Sets the downloader for this PP."""
2032 self._downloader = downloader
2034 def run(self, information):
2035 """Run the PostProcessor.
2037 The "information" argument is a dictionary like the ones
2038 composed by InfoExtractors. The only difference is that this
2039 one has an extra field called "filepath" that points to the
2042 When this method returns None, the postprocessing chain is
2043 stopped. However, this method may return an information
2044 dictionary that will be passed to the next postprocessing
2045 object in the chain. It can be the one it received after
2046 changing some fields.
2048 In addition, this method may raise a PostProcessingError
2049 exception that will be taken into account by the downloader
2052 return information # by default, do nothing
2054 ### MAIN PROGRAM ###
2055 if __name__ == '__main__':
2057 # Modules needed only when running the main program
2061 # Function to update the program file with the latest version from bitbucket.org
2062 def update_self(downloader, filename):
2063 # Note: downloader only used for options
2064 if not os.access (filename, os.W_OK):
2065 sys.exit('ERROR: no write permissions on %s' % filename)
2067 downloader.to_stdout('Updating to latest stable version...')
2068 latest_url = 'http://bitbucket.org/rg3/youtube-dl/raw/tip/LATEST_VERSION'
2069 latest_version = urllib.urlopen(latest_url).read().strip()
2070 prog_url = 'http://bitbucket.org/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
2071 newcontent = urllib.urlopen(prog_url).read()
2072 stream = open(filename, 'w')
2073 stream.write(newcontent)
2075 downloader.to_stdout('Updated to version %s' % latest_version)
2077 # General configuration
2078 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
2079 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
2080 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
2082 # Parse command line
2083 parser = optparse.OptionParser(
2084 usage='Usage: %prog [options] url...',
2085 version='2010.06.06',
2086 conflict_handler='resolve',
2089 parser.add_option('-h', '--help',
2090 action='help', help='print this help text and exit')
2091 parser.add_option('-v', '--version',
2092 action='version', help='print program version and exit')
2093 parser.add_option('-U', '--update',
2094 action='store_true', dest='update_self', help='update this program to latest stable version')
2095 parser.add_option('-i', '--ignore-errors',
2096 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
2097 parser.add_option('-r', '--rate-limit',
2098 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
2099 parser.add_option('-R', '--retries',
2100 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
2102 authentication = optparse.OptionGroup(parser, 'Authentication Options')
2103 authentication.add_option('-u', '--username',
2104 dest='username', metavar='USERNAME', help='account username')
2105 authentication.add_option('-p', '--password',
2106 dest='password', metavar='PASSWORD', help='account password')
2107 authentication.add_option('-n', '--netrc',
2108 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
2109 parser.add_option_group(authentication)
2111 video_format = optparse.OptionGroup(parser, 'Video Format Options')
2112 video_format.add_option('-f', '--format',
2113 action='store', dest='format', metavar='FORMAT', help='video format code')
2114 video_format.add_option('-b', '--best-quality',
2115 action='store_const', dest='format', help='download the best quality video possible', const='0')
2116 video_format.add_option('-m', '--mobile-version',
2117 action='store_const', dest='format', help='alias for -f 17', const='17')
2118 video_format.add_option('-d', '--high-def',
2119 action='store_const', dest='format', help='alias for -f 22', const='22')
2120 video_format.add_option('--all-formats',
2121 action='store_const', dest='format', help='download all available video formats', const='-1')
2122 video_format.add_option('--max-quality',
2123 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format limit for -b')
2124 parser.add_option_group(video_format)
2126 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
2127 verbosity.add_option('-q', '--quiet',
2128 action='store_true', dest='quiet', help='activates quiet mode', default=False)
2129 verbosity.add_option('-s', '--simulate',
2130 action='store_true', dest='simulate', help='do not download video', default=False)
2131 verbosity.add_option('-g', '--get-url',
2132 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
2133 verbosity.add_option('-e', '--get-title',
2134 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
2135 verbosity.add_option('--get-thumbnail',
2136 action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False)
2137 verbosity.add_option('--get-description',
2138 action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
2139 verbosity.add_option('--no-progress',
2140 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
2141 parser.add_option_group(verbosity)
2143 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
2144 filesystem.add_option('-t', '--title',
2145 action='store_true', dest='usetitle', help='use title in file name', default=False)
2146 filesystem.add_option('-l', '--literal',
2147 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
2148 filesystem.add_option('-o', '--output',
2149 dest='outtmpl', metavar='TEMPLATE', help='output filename template')
2150 filesystem.add_option('-a', '--batch-file',
2151 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
2152 filesystem.add_option('-w', '--no-overwrites',
2153 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
2154 filesystem.add_option('-c', '--continue',
2155 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
2156 parser.add_option_group(filesystem)
2158 (opts, args) = parser.parse_args()
2160 # Batch file verification
2162 if opts.batchfile is not None:
2164 if opts.batchfile == '-':
2167 batchfd = open(opts.batchfile, 'r')
2168 batchurls = batchfd.readlines()
2169 batchurls = [x.strip() for x in batchurls]
2170 batchurls = [x for x in batchurls if len(x) > 0]
2172 sys.exit(u'ERROR: batch file could not be read')
2173 all_urls = batchurls + args
2175 # Conflicting, missing and erroneous options
2176 if opts.usenetrc and (opts.username is not None or opts.password is not None):
2177 parser.error(u'using .netrc conflicts with giving username/password')
2178 if opts.password is not None and opts.username is None:
2179 parser.error(u'account username missing')
2180 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
2181 parser.error(u'using output template conflicts with using title or literal title')
2182 if opts.usetitle and opts.useliteral:
2183 parser.error(u'using title conflicts with using literal title')
2184 if opts.username is not None and opts.password is None:
2185 opts.password = getpass.getpass(u'Type account password and press return:')
2186 if opts.ratelimit is not None:
2187 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
2188 if numeric_limit is None:
2189 parser.error(u'invalid rate limit specified')
2190 opts.ratelimit = numeric_limit
2191 if opts.retries is not None:
2193 opts.retries = long(opts.retries)
2194 except (TypeError, ValueError), err:
2195 parser.error(u'invalid retry count specified')
2197 # Information extractors
2198 youtube_ie = YoutubeIE()
2199 metacafe_ie = MetacafeIE(youtube_ie)
2200 dailymotion_ie = DailymotionIE()
2201 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
2202 youtube_user_ie = YoutubeUserIE(youtube_ie)
2203 youtube_search_ie = YoutubeSearchIE(youtube_ie)
2204 google_ie = GoogleIE()
2205 google_search_ie = GoogleSearchIE(google_ie)
2206 photobucket_ie = PhotobucketIE()
2207 yahoo_ie = YahooIE()
2208 yahoo_search_ie = YahooSearchIE(yahoo_ie)
2209 generic_ie = GenericIE()
2212 fd = FileDownloader({
2213 'usenetrc': opts.usenetrc,
2214 'username': opts.username,
2215 'password': opts.password,
2216 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2217 'forceurl': opts.geturl,
2218 'forcetitle': opts.gettitle,
2219 'forcethumbnail': opts.getthumbnail,
2220 'forcedescription': opts.getdescription,
2221 'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2222 'format': opts.format,
2223 'format_limit': opts.format_limit,
2224 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
2225 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
2226 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
2227 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
2228 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
2229 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
2230 or u'%(id)s.%(ext)s'),
2231 'ignoreerrors': opts.ignoreerrors,
2232 'ratelimit': opts.ratelimit,
2233 'nooverwrites': opts.nooverwrites,
2234 'retries': opts.retries,
2235 'continuedl': opts.continue_dl,
2236 'noprogress': opts.noprogress,
2238 fd.add_info_extractor(youtube_search_ie)
2239 fd.add_info_extractor(youtube_pl_ie)
2240 fd.add_info_extractor(youtube_user_ie)
2241 fd.add_info_extractor(metacafe_ie)
2242 fd.add_info_extractor(dailymotion_ie)
2243 fd.add_info_extractor(youtube_ie)
2244 fd.add_info_extractor(google_ie)
2245 fd.add_info_extractor(google_search_ie)
2246 fd.add_info_extractor(photobucket_ie)
2247 fd.add_info_extractor(yahoo_ie)
2248 fd.add_info_extractor(yahoo_search_ie)
2250 # This must come last since it's the
2251 # fallback if none of the others work
2252 fd.add_info_extractor(generic_ie)
2255 if opts.update_self:
2256 update_self(fd, sys.argv[0])
2259 if len(all_urls) < 1:
2260 if not opts.update_self:
2261 parser.error(u'you must provide at least one URL')
2264 retcode = fd.download(all_urls)
2267 except DownloadError:
2269 except SameFileError:
2270 sys.exit(u'ERROR: fixed output name but more than one file to download')
2271 except KeyboardInterrupt:
2272 sys.exit(u'\nERROR: Interrupted by user')