2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # Author: Benjamin Johnson
6 # License: Public domain code
23 # parse_qs was moved from the cgi module to the urlparse module recently.
25 from urlparse import parse_qs
27 from cgi import parse_qs
30 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.6) Gecko/20100627 Firefox/3.6.6',
31 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
32 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
33 'Accept-Language': 'en-us,en;q=0.5',
36 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
38 def preferredencoding():
39 """Get preferred encoding.
41 Returns the best encoding scheme for the system, based on
42 locale.getpreferredencoding() and some further tweaks.
44 def yield_preferredencoding():
46 pref = locale.getpreferredencoding()
52 return yield_preferredencoding().next()
54 def htmlentity_transform(matchobj):
55 """Transforms an HTML entity to a Unicode character.
57 This function receives a match object and is intended to be used with
58 the re.sub() function.
60 entity = matchobj.group(1)
62 # Known non-numeric HTML entity
63 if entity in htmlentitydefs.name2codepoint:
64 return unichr(htmlentitydefs.name2codepoint[entity])
67 mobj = re.match(ur'(?u)#(x?\d+)', entity)
69 numstr = mobj.group(1)
70 if numstr.startswith(u'x'):
72 numstr = u'0%s' % numstr
75 return unichr(long(numstr, base))
77 # Unknown entity in name, return its literal representation
78 return (u'&%s;' % entity)
80 def sanitize_title(utitle):
81 """Sanitizes a video title so it could be used as part of a filename."""
82 utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
83 return utitle.replace(unicode(os.sep), u'%')
85 def sanitize_open(filename, open_mode):
86 """Try to open the given filename, and slightly tweak it if this fails.
88 Attempts to open the given filename. If this fails, it tries to change
89 the filename slightly, step by step, until it's either able to open it
90 or it fails and raises a final exception, like the standard open()
93 It returns the tuple (stream, definitive_file_name).
97 return (sys.stdout, filename)
98 stream = open(filename, open_mode)
99 return (stream, filename)
100 except (IOError, OSError), err:
101 # In case of error, try to remove win32 forbidden chars
102 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
104 # An exception here should be caught in the caller
105 stream = open(filename, open_mode)
106 return (stream, filename)
109 class DownloadError(Exception):
110 """Download Error exception.
112 This exception may be thrown by FileDownloader objects if they are not
113 configured to continue on errors. They will contain the appropriate
118 class SameFileError(Exception):
119 """Same File exception.
121 This exception will be thrown by FileDownloader objects if they detect
122 multiple files would have to be downloaded to the same file on disk.
126 class PostProcessingError(Exception):
127 """Post Processing exception.
129 This exception may be raised by PostProcessor's .run() method to
130 indicate an error in the postprocessing task.
134 class UnavailableVideoError(Exception):
135 """Unavailable Format exception.
137 This exception will be thrown when a video is requested
138 in a format that is not available for that video.
142 class ContentTooShortError(Exception):
143 """Content Too Short exception.
145 This exception may be raised by FileDownloader objects when a file they
146 download is too small for what the server announced first, indicating
147 the connection was probably interrupted.
153 def __init__(self, downloaded, expected):
154 self.downloaded = downloaded
155 self.expected = expected
157 class FileDownloader(object):
158 """File Downloader class.
160 File downloader objects are the ones responsible of downloading the
161 actual video file and writing it to disk if the user has requested
162 it, among some other tasks. In most cases there should be one per
163 program. As, given a video URL, the downloader doesn't know how to
164 extract all the needed information, task that InfoExtractors do, it
165 has to pass the URL to one of them.
167 For this, file downloader objects have a method that allows
168 InfoExtractors to be registered in a given order. When it is passed
169 a URL, the file downloader handles it to the first InfoExtractor it
170 finds that reports being able to handle it. The InfoExtractor extracts
171 all the information about the video or videos the URL refers to, and
172 asks the FileDownloader to process the video information, possibly
173 downloading the video.
175 File downloaders accept a lot of parameters. In order not to saturate
176 the object constructor with arguments, it receives a dictionary of
177 options instead. These options are available through the params
178 attribute for the InfoExtractors to use. The FileDownloader also
179 registers itself as the downloader in charge for the InfoExtractors
180 that are added to it, so this is a "mutual registration".
184 username: Username for authentication purposes.
185 password: Password for authentication purposes.
186 usenetrc: Use netrc for authentication instead.
187 quiet: Do not print messages to stdout.
188 forceurl: Force printing final URL.
189 forcetitle: Force printing title.
190 simulate: Do not download the video files.
191 format: Video format code.
192 format_limit: Highest quality format to try.
193 outtmpl: Template for output names.
194 ignoreerrors: Do not stop on download errors.
195 ratelimit: Download speed limit, in bytes/sec.
196 nooverwrites: Prevent overwriting files.
197 retries: Number of times to retry for HTTP error 503
198 continuedl: Try to continue downloads if possible.
199 noprogress: Do not print the progress bar.
205 _download_retcode = None
206 _num_downloads = None
208 def __init__(self, params):
209 """Create a FileDownloader object with the given options."""
212 self._download_retcode = 0
213 self._num_downloads = 0
217 def pmkdir(filename):
218 """Create directory components in filename. Similar to Unix "mkdir -p"."""
219 components = filename.split(os.sep)
220 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
221 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
222 for dir in aggregate:
223 if not os.path.exists(dir):
227 def format_bytes(bytes):
230 if type(bytes) is str:
235 exponent = long(math.log(bytes, 1024.0))
236 suffix = 'bkMGTPEZY'[exponent]
237 converted = float(bytes) / float(1024**exponent)
238 return '%.2f%s' % (converted, suffix)
241 def calc_percent(byte_counter, data_len):
244 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
247 def calc_eta(start, now, total, current):
251 if current == 0 or dif < 0.001: # One millisecond
253 rate = float(current) / dif
254 eta = long((float(total) - float(current)) / rate)
255 (eta_mins, eta_secs) = divmod(eta, 60)
258 return '%02d:%02d' % (eta_mins, eta_secs)
261 def calc_speed(start, now, bytes):
263 if bytes == 0 or dif < 0.001: # One millisecond
264 return '%10s' % '---b/s'
265 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
268 def best_block_size(elapsed_time, bytes):
269 new_min = max(bytes / 2.0, 1.0)
270 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
271 if elapsed_time < 0.001:
273 rate = bytes / elapsed_time
281 def parse_bytes(bytestr):
282 """Parse a string indicating a byte quantity into a long integer."""
283 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
286 number = float(matchobj.group(1))
287 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
288 return long(round(number * multiplier))
292 """Verify a URL is valid and data could be downloaded. Return real data URL."""
293 request = urllib2.Request(url, None, std_headers)
294 data = urllib2.urlopen(request)
300 def add_info_extractor(self, ie):
301 """Add an InfoExtractor object to the end of the list."""
303 ie.set_downloader(self)
305 def add_post_processor(self, pp):
306 """Add a PostProcessor object to the end of the chain."""
308 pp.set_downloader(self)
310 def to_stdout(self, message, skip_eol=False, ignore_encoding_errors=False):
311 """Print message to stdout if not in quiet mode."""
313 if not self.params.get('quiet', False):
314 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
316 except (UnicodeEncodeError), err:
317 if not ignore_encoding_errors:
320 def to_stderr(self, message):
321 """Print message to stderr."""
322 print >>sys.stderr, message.encode(preferredencoding())
324 def fixed_template(self):
325 """Checks if the output template is fixed."""
326 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
328 def trouble(self, message=None):
329 """Determine action to take when a download problem appears.
331 Depending on if the downloader has been configured to ignore
332 download errors or not, this method may throw an exception or
333 not when errors are found, after printing the message.
335 if message is not None:
336 self.to_stderr(message)
337 if not self.params.get('ignoreerrors', False):
338 raise DownloadError(message)
339 self._download_retcode = 1
341 def slow_down(self, start_time, byte_counter):
342 """Sleep if the download speed is over the rate limit."""
343 rate_limit = self.params.get('ratelimit', None)
344 if rate_limit is None or byte_counter == 0:
347 elapsed = now - start_time
350 speed = float(byte_counter) / elapsed
351 if speed > rate_limit:
352 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
354 def report_destination(self, filename):
355 """Report destination filename."""
356 self.to_stdout(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
358 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
359 """Report download progress."""
360 if self.params.get('noprogress', False):
362 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
363 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
365 def report_resuming_byte(self, resume_len):
366 """Report attemtp to resume at given byte."""
367 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
369 def report_retry(self, count, retries):
370 """Report retry in case of HTTP error 503"""
371 self.to_stdout(u'[download] Got HTTP error 503. Retrying (attempt %d of %d)...' % (count, retries))
373 def report_file_already_downloaded(self, file_name):
374 """Report file has already been fully downloaded."""
376 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
377 except (UnicodeEncodeError), err:
378 self.to_stdout(u'[download] The file has already been downloaded')
380 def report_unable_to_resume(self):
381 """Report it was impossible to resume download."""
382 self.to_stdout(u'[download] Unable to resume')
384 def report_finish(self):
385 """Report download finished."""
386 if self.params.get('noprogress', False):
387 self.to_stdout(u'[download] Download completed')
391 def increment_downloads(self):
392 """Increment the ordinal that assigns a number to each file."""
393 self._num_downloads += 1
395 def process_info(self, info_dict):
396 """Process a single dictionary returned by an InfoExtractor."""
397 # Do nothing else if in simulate mode
398 if self.params.get('simulate', False):
399 # Verify URL if it's an HTTP one
400 if info_dict['url'].startswith('http'):
402 self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8')
403 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
404 raise UnavailableVideoError
407 if self.params.get('forcetitle', False):
408 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
409 if self.params.get('forceurl', False):
410 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
411 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
412 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
413 if self.params.get('forcedescription', False) and 'description' in info_dict:
414 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
419 template_dict = dict(info_dict)
420 template_dict['epoch'] = unicode(long(time.time()))
421 template_dict['ord'] = unicode('%05d' % self._num_downloads)
422 filename = self.params['outtmpl'] % template_dict
423 except (ValueError, KeyError), err:
424 self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
425 if self.params.get('nooverwrites', False) and os.path.exists(filename):
426 self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
430 self.pmkdir(filename)
431 except (OSError, IOError), err:
432 self.trouble('ERROR: unable to create directories: %s' % str(err))
436 success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
437 except (OSError, IOError), err:
438 raise UnavailableVideoError
439 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
440 self.trouble('ERROR: unable to download video data: %s' % str(err))
442 except (ContentTooShortError, ), err:
443 self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
448 self.post_process(filename, info_dict)
449 except (PostProcessingError), err:
450 self.trouble('ERROR: postprocessing: %s' % str(err))
453 def download(self, url_list):
454 """Download a given list of URLs."""
455 if len(url_list) > 1 and self.fixed_template():
456 raise SameFileError(self.params['outtmpl'])
459 suitable_found = False
461 # Go to next InfoExtractor if not suitable
462 if not ie.suitable(url):
465 # Suitable InfoExtractor found
466 suitable_found = True
468 # Extract information from URL and process it
471 # Suitable InfoExtractor had been found; go to next URL
474 if not suitable_found:
475 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
477 return self._download_retcode
479 def post_process(self, filename, ie_info):
480 """Run the postprocessing chain on the given file."""
482 info['filepath'] = filename
488 def _download_with_rtmpdump(self, filename, url, player_url):
489 self.report_destination(filename)
491 # Check for rtmpdump first
493 subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
494 except (OSError, IOError):
495 self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
498 # Download using rtmpdump. rtmpdump returns exit code 2 when
499 # the connection was interrumpted and resuming appears to be
500 # possible. This is part of rtmpdump's normal usage, AFAIK.
501 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename]
502 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
503 while retval == 2 or retval == 1:
504 prevsize = os.path.getsize(filename)
505 self.to_stdout(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
506 time.sleep(5.0) # This seems to be needed
507 retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
508 cursize = os.path.getsize(filename)
509 if prevsize == cursize and retval == 1:
512 self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
515 self.trouble('\nERROR: rtmpdump exited with code %d' % retval)
518 def _do_download(self, filename, url, player_url):
519 # Attempt to download using rtmpdump
520 if url.startswith('rtmp'):
521 return self._download_with_rtmpdump(filename, url, player_url)
525 basic_request = urllib2.Request(url, None, std_headers)
526 request = urllib2.Request(url, None, std_headers)
528 # Establish possible resume length
529 if os.path.isfile(filename):
530 resume_len = os.path.getsize(filename)
534 # Request parameters in case of being able to resume
535 if self.params.get('continuedl', False) and resume_len != 0:
536 self.report_resuming_byte(resume_len)
537 request.add_header('Range','bytes=%d-' % resume_len)
541 retries = self.params.get('retries', 0)
543 # Establish connection
545 data = urllib2.urlopen(request)
547 except (urllib2.HTTPError, ), err:
549 # Retry in case of HTTP error 503
552 self.report_retry(count, retries)
554 if err.code != 416: # 416 is 'Requested range not satisfiable'
557 data = urllib2.urlopen(basic_request)
558 content_length = data.info()['Content-Length']
560 if content_length is not None and long(content_length) == resume_len:
561 # Because the file had already been fully downloaded
562 self.report_file_already_downloaded(filename)
565 # Because the server didn't let us
566 self.report_unable_to_resume()
569 data_len = data.info().get('Content-length', None)
570 data_len_str = self.format_bytes(data_len)
577 data_block = data.read(block_size)
579 data_block_len = len(data_block)
580 if data_block_len == 0:
582 byte_counter += data_block_len
584 # Open file just in time
587 (stream, filename) = sanitize_open(filename, open_mode)
588 self.report_destination(filename)
589 except (OSError, IOError), err:
590 self.trouble('ERROR: unable to open for writing: %s' % str(err))
593 stream.write(data_block)
594 except (IOError, OSError), err:
595 self.trouble('\nERROR: unable to write data: %s' % str(err))
596 block_size = self.best_block_size(after - before, data_block_len)
599 percent_str = self.calc_percent(byte_counter, data_len)
600 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
601 speed_str = self.calc_speed(start, time.time(), byte_counter)
602 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
605 self.slow_down(start, byte_counter)
608 if data_len is not None and str(byte_counter) != data_len:
609 raise ContentTooShortError(byte_counter, long(data_len))
612 class InfoExtractor(object):
613 """Information Extractor class.
615 Information extractors are the classes that, given a URL, extract
616 information from the video (or videos) the URL refers to. This
617 information includes the real video URL, the video title and simplified
618 title, author and others. The information is stored in a dictionary
619 which is then passed to the FileDownloader. The FileDownloader
620 processes this information possibly downloading the video to the file
621 system, among other possible outcomes. The dictionaries must include
622 the following fields:
624 id: Video identifier.
625 url: Final video URL.
626 uploader: Nickname of the video uploader.
627 title: Literal title.
628 stitle: Simplified title.
629 ext: Video filename extension.
630 format: Video format.
631 player_url: SWF Player URL (may be None).
633 The following fields are optional. Their primary purpose is to allow
634 youtube-dl to serve as the backend for a video search function, such
635 as the one in youtube2mp3. They are only used when their respective
636 forced printing functions are called:
638 thumbnail: Full URL to a video thumbnail image.
639 description: One-line video description.
641 Subclasses of this one should re-define the _real_initialize() and
642 _real_extract() methods, as well as the suitable() static method.
643 Probably, they should also be instantiated and added to the main
650 def __init__(self, downloader=None):
651 """Constructor. Receives an optional downloader."""
653 self.set_downloader(downloader)
657 """Receives a URL and returns True if suitable for this IE."""
660 def initialize(self):
661 """Initializes an instance (authentication, etc)."""
663 self._real_initialize()
666 def extract(self, url):
667 """Extracts URL information and returns it in list of dicts."""
669 return self._real_extract(url)
671 def set_downloader(self, downloader):
672 """Sets the downloader for this IE."""
673 self._downloader = downloader
675 def _real_initialize(self):
676 """Real initialization process. Redefine in subclasses."""
679 def _real_extract(self, url):
680 """Real extraction process. Redefine in subclasses."""
683 class YoutubeIE(InfoExtractor):
684 """Information extractor for youtube.com."""
686 _VALID_URL = r'^((?:http://)?(?:youtu\.be/|(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?[\?#](?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
687 _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
688 _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
689 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
690 _NETRC_MACHINE = 'youtube'
691 # Listed in order of priority for the -b option
692 _available_formats = ['38', '37', '22', '45', '35', '34', '43', '18', '6', '5', '17', '13', None]
693 _video_extensions = {
699 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
706 return (re.match(YoutubeIE._VALID_URL, url) is not None)
708 def report_lang(self):
709 """Report attempt to set language."""
710 self._downloader.to_stdout(u'[youtube] Setting language')
712 def report_login(self):
713 """Report attempt to log in."""
714 self._downloader.to_stdout(u'[youtube] Logging in')
716 def report_age_confirmation(self):
717 """Report attempt to confirm age."""
718 self._downloader.to_stdout(u'[youtube] Confirming age')
720 def report_video_webpage_download(self, video_id):
721 """Report attempt to download video webpage."""
722 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
724 def report_video_info_webpage_download(self, video_id):
725 """Report attempt to download video info webpage."""
726 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
728 def report_information_extraction(self, video_id):
729 """Report attempt to extract video information."""
730 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
732 def report_unavailable_format(self, video_id, format):
733 """Report extracted video URL."""
734 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
736 def report_rtmp_download(self):
737 """Indicate the download will use the RTMP protocol."""
738 self._downloader.to_stdout(u'[youtube] RTMP download detected')
740 def _real_initialize(self):
741 if self._downloader is None:
746 downloader_params = self._downloader.params
748 # Attempt to use provided username and password or .netrc data
749 if downloader_params.get('username', None) is not None:
750 username = downloader_params['username']
751 password = downloader_params['password']
752 elif downloader_params.get('usenetrc', False):
754 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
759 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
760 except (IOError, netrc.NetrcParseError), err:
761 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
765 request = urllib2.Request(self._LANG_URL, None, std_headers)
768 urllib2.urlopen(request).read()
769 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
770 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
773 # No authentication to be performed
779 'current_form': 'loginForm',
781 'action_login': 'Log In',
782 'username': username,
783 'password': password,
785 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
788 login_results = urllib2.urlopen(request).read()
789 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
790 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
792 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
793 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
799 'action_confirm': 'Confirm',
801 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
803 self.report_age_confirmation()
804 age_results = urllib2.urlopen(request).read()
805 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
806 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
809 def _real_extract(self, url):
810 # Extract video id from URL
811 mobj = re.match(self._VALID_URL, url)
813 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
816 # At this point we have a new video
817 if self._downloader is not None:
818 self._downloader.increment_downloads()
819 video_id = mobj.group(2)
821 # Downloader parameters
826 if self._downloader is not None:
827 params = self._downloader.params
828 format_param = params.get('format', None)
829 if format_param == '0':
830 format_limit = params.get('format_limit', None)
831 if format_limit is not None:
833 # Start at a different format if the user has limited the maximum quality
834 quality_index = self._available_formats.index(format_limit)
837 format_param = self._available_formats[quality_index]
839 elif format_param == '-1':
840 format_param = self._available_formats[quality_index]
845 video_extension = self._video_extensions.get(format_param, 'flv')
848 self.report_video_webpage_download(video_id)
849 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id, None, std_headers)
851 video_webpage = urllib2.urlopen(request).read()
852 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
853 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
856 # Attempt to extract SWF player URL
857 mobj = re.search(r'swfConfig.*"(http://.*?watch.*?-.*?\.swf)"', video_webpage)
859 player_url = mobj.group(1)
864 self.report_video_info_webpage_download(video_id)
865 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
866 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
867 % (video_id, el_type))
868 request = urllib2.Request(video_info_url, None, std_headers)
870 video_info_webpage = urllib2.urlopen(request).read()
871 video_info = parse_qs(video_info_webpage)
872 if 'token' in video_info:
874 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
875 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
877 self.report_information_extraction(video_id)
880 if 'token' not in video_info:
881 # Attempt to see if YouTube has issued an error message
882 if 'reason' not in video_info:
883 self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
884 stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
885 stream.write(video_info_webpage)
888 reason = urllib.unquote_plus(video_info['reason'][0])
889 self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
891 token = urllib.unquote_plus(video_info['token'][0])
892 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
893 if format_param is not None:
894 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
896 # Check possible RTMP download
897 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
898 self.report_rtmp_download()
899 video_real_url = video_info['conn'][0]
902 if 'author' not in video_info:
903 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
905 video_uploader = urllib.unquote_plus(video_info['author'][0])
908 if 'title' not in video_info:
909 self._downloader.trouble(u'ERROR: unable to extract video title')
911 video_title = urllib.unquote_plus(video_info['title'][0])
912 video_title = video_title.decode('utf-8')
913 video_title = sanitize_title(video_title)
916 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
917 simple_title = simple_title.strip(ur'_')
920 if 'thumbnail_url' not in video_info:
921 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
923 else: # don't panic if we can't find it
924 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
927 video_description = 'No description available.'
928 if self._downloader.params.get('forcedescription', False):
929 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
931 video_description = mobj.group(1)
934 # Process video information
935 self._downloader.process_info({
936 'id': video_id.decode('utf-8'),
937 'url': video_real_url.decode('utf-8'),
938 'uploader': video_uploader.decode('utf-8'),
939 'title': video_title,
940 'stitle': simple_title,
941 'ext': video_extension.decode('utf-8'),
942 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
943 'thumbnail': video_thumbnail.decode('utf-8'),
944 'description': video_description.decode('utf-8'),
945 'player_url': player_url,
950 if quality_index == len(self._available_formats):
954 format_param = self._available_formats[quality_index]
958 except UnavailableFormatError, err:
959 if best_quality or all_formats:
961 if quality_index == len(self._available_formats):
962 # I don't ever expect this to happen
964 self._downloader.trouble(u'ERROR: no known formats available for video')
967 self.report_unavailable_format(video_id, format_param)
968 format_param = self._available_formats[quality_index]
971 self._downloader.trouble('ERROR: format not available for video')
975 class MetacafeIE(InfoExtractor):
976 """Information Extractor for metacafe.com."""
978 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
979 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
980 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
983 def __init__(self, youtube_ie, downloader=None):
984 InfoExtractor.__init__(self, downloader)
985 self._youtube_ie = youtube_ie
989 return (re.match(MetacafeIE._VALID_URL, url) is not None)
991 def report_disclaimer(self):
992 """Report disclaimer retrieval."""
993 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
995 def report_age_confirmation(self):
996 """Report attempt to confirm age."""
997 self._downloader.to_stdout(u'[metacafe] Confirming age')
999 def report_download_webpage(self, video_id):
1000 """Report webpage download."""
1001 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
1003 def report_extraction(self, video_id):
1004 """Report information extraction."""
1005 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
1007 def _real_initialize(self):
1008 # Retrieve disclaimer
1009 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
1011 self.report_disclaimer()
1012 disclaimer = urllib2.urlopen(request).read()
1013 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1014 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1020 'submit': "Continue - I'm over 18",
1022 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
1024 self.report_age_confirmation()
1025 disclaimer = urllib2.urlopen(request).read()
1026 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1027 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1030 def _real_extract(self, url):
1031 # Extract id and simplified title from URL
1032 mobj = re.match(self._VALID_URL, url)
1034 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1037 video_id = mobj.group(1)
1039 # Check if video comes from YouTube
1040 mobj2 = re.match(r'^yt-(.*)$', video_id)
1041 if mobj2 is not None:
1042 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1045 # At this point we have a new video
1046 self._downloader.increment_downloads()
1048 simple_title = mobj.group(2).decode('utf-8')
1049 video_extension = 'flv'
1051 # Retrieve video webpage to extract further information
1052 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1054 self.report_download_webpage(video_id)
1055 webpage = urllib2.urlopen(request).read()
1056 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1057 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1060 # Extract URL, uploader and title from webpage
1061 self.report_extraction(video_id)
1062 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1064 self._downloader.trouble(u'ERROR: unable to extract media URL')
1066 mediaURL = urllib.unquote(mobj.group(1))
1068 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1070 # self._downloader.trouble(u'ERROR: unable to extract gdaKey')
1072 #gdaKey = mobj.group(1)
1074 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1076 video_url = mediaURL
1078 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1080 self._downloader.trouble(u'ERROR: unable to extract title')
1082 video_title = mobj.group(1).decode('utf-8')
1083 video_title = sanitize_title(video_title)
1085 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1087 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1089 video_uploader = mobj.group(1)
1092 # Process video information
1093 self._downloader.process_info({
1094 'id': video_id.decode('utf-8'),
1095 'url': video_url.decode('utf-8'),
1096 'uploader': video_uploader.decode('utf-8'),
1097 'title': video_title,
1098 'stitle': simple_title,
1099 'ext': video_extension.decode('utf-8'),
1103 except UnavailableVideoError:
1104 self._downloader.trouble(u'ERROR: unable to download video')
1107 class DailymotionIE(InfoExtractor):
1108 """Information Extractor for Dailymotion"""
1110 _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1112 def __init__(self, downloader=None):
1113 InfoExtractor.__init__(self, downloader)
1117 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1119 def report_download_webpage(self, video_id):
1120 """Report webpage download."""
1121 self._downloader.to_stdout(u'[dailymotion] %s: Downloading webpage' % video_id)
1123 def report_extraction(self, video_id):
1124 """Report information extraction."""
1125 self._downloader.to_stdout(u'[dailymotion] %s: Extracting information' % video_id)
1127 def _real_initialize(self):
1130 def _real_extract(self, url):
1131 # Extract id and simplified title from URL
1132 mobj = re.match(self._VALID_URL, url)
1134 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1137 # At this point we have a new video
1138 self._downloader.increment_downloads()
1139 video_id = mobj.group(1)
1141 simple_title = mobj.group(2).decode('utf-8')
1142 video_extension = 'flv'
1144 # Retrieve video webpage to extract further information
1145 request = urllib2.Request(url)
1147 self.report_download_webpage(video_id)
1148 webpage = urllib2.urlopen(request).read()
1149 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1150 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1153 # Extract URL, uploader and title from webpage
1154 self.report_extraction(video_id)
1155 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1157 self._downloader.trouble(u'ERROR: unable to extract media URL')
1159 mediaURL = urllib.unquote(mobj.group(1))
1161 # if needed add http://www.dailymotion.com/ if relative URL
1163 video_url = mediaURL
1165 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1166 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1168 self._downloader.trouble(u'ERROR: unable to extract title')
1170 video_title = mobj.group(1).decode('utf-8')
1171 video_title = sanitize_title(video_title)
1173 mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a></div>', webpage)
1175 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1177 video_uploader = mobj.group(1)
1180 # Process video information
1181 self._downloader.process_info({
1182 'id': video_id.decode('utf-8'),
1183 'url': video_url.decode('utf-8'),
1184 'uploader': video_uploader.decode('utf-8'),
1185 'title': video_title,
1186 'stitle': simple_title,
1187 'ext': video_extension.decode('utf-8'),
1191 except UnavailableVideoError:
1192 self._downloader.trouble(u'ERROR: unable to download video')
1194 class GoogleIE(InfoExtractor):
1195 """Information extractor for video.google.com."""
1197 _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1199 def __init__(self, downloader=None):
1200 InfoExtractor.__init__(self, downloader)
1204 return (re.match(GoogleIE._VALID_URL, url) is not None)
1206 def report_download_webpage(self, video_id):
1207 """Report webpage download."""
1208 self._downloader.to_stdout(u'[video.google] %s: Downloading webpage' % video_id)
1210 def report_extraction(self, video_id):
1211 """Report information extraction."""
1212 self._downloader.to_stdout(u'[video.google] %s: Extracting information' % video_id)
1214 def _real_initialize(self):
1217 def _real_extract(self, url):
1218 # Extract id from URL
1219 mobj = re.match(self._VALID_URL, url)
1221 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1224 # At this point we have a new video
1225 self._downloader.increment_downloads()
1226 video_id = mobj.group(1)
1228 video_extension = 'mp4'
1230 # Retrieve video webpage to extract further information
1231 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1233 self.report_download_webpage(video_id)
1234 webpage = urllib2.urlopen(request).read()
1235 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1236 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1239 # Extract URL, uploader, and title from webpage
1240 self.report_extraction(video_id)
1241 mobj = re.search(r"download_url:'([^']+)'", webpage)
1243 video_extension = 'flv'
1244 mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1246 self._downloader.trouble(u'ERROR: unable to extract media URL')
1248 mediaURL = urllib.unquote(mobj.group(1))
1249 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1250 mediaURL = mediaURL.replace('\\x26', '\x26')
1252 video_url = mediaURL
1254 mobj = re.search(r'<title>(.*)</title>', webpage)
1256 self._downloader.trouble(u'ERROR: unable to extract title')
1258 video_title = mobj.group(1).decode('utf-8')
1259 video_title = sanitize_title(video_title)
1260 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1262 # Extract video description
1263 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1265 self._downloader.trouble(u'ERROR: unable to extract video description')
1267 video_description = mobj.group(1).decode('utf-8')
1268 if not video_description:
1269 video_description = 'No description available.'
1271 # Extract video thumbnail
1272 if self._downloader.params.get('forcethumbnail', False):
1273 request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1275 webpage = urllib2.urlopen(request).read()
1276 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1277 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1279 mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1281 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1283 video_thumbnail = mobj.group(1)
1284 else: # we need something to pass to process_info
1285 video_thumbnail = ''
1289 # Process video information
1290 self._downloader.process_info({
1291 'id': video_id.decode('utf-8'),
1292 'url': video_url.decode('utf-8'),
1294 'title': video_title,
1295 'stitle': simple_title,
1296 'ext': video_extension.decode('utf-8'),
1300 except UnavailableVideoError:
1301 self._downloader.trouble(u'ERROR: unable to download video')
1304 class PhotobucketIE(InfoExtractor):
1305 """Information extractor for photobucket.com."""
1307 _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1309 def __init__(self, downloader=None):
1310 InfoExtractor.__init__(self, downloader)
1314 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1316 def report_download_webpage(self, video_id):
1317 """Report webpage download."""
1318 self._downloader.to_stdout(u'[photobucket] %s: Downloading webpage' % video_id)
1320 def report_extraction(self, video_id):
1321 """Report information extraction."""
1322 self._downloader.to_stdout(u'[photobucket] %s: Extracting information' % video_id)
1324 def _real_initialize(self):
1327 def _real_extract(self, url):
1328 # Extract id from URL
1329 mobj = re.match(self._VALID_URL, url)
1331 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1334 # At this point we have a new video
1335 self._downloader.increment_downloads()
1336 video_id = mobj.group(1)
1338 video_extension = 'flv'
1340 # Retrieve video webpage to extract further information
1341 request = urllib2.Request(url)
1343 self.report_download_webpage(video_id)
1344 webpage = urllib2.urlopen(request).read()
1345 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1346 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1349 # Extract URL, uploader, and title from webpage
1350 self.report_extraction(video_id)
1351 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1353 self._downloader.trouble(u'ERROR: unable to extract media URL')
1355 mediaURL = urllib.unquote(mobj.group(1))
1357 video_url = mediaURL
1359 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1361 self._downloader.trouble(u'ERROR: unable to extract title')
1363 video_title = mobj.group(1).decode('utf-8')
1364 video_title = sanitize_title(video_title)
1365 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1367 video_uploader = mobj.group(2).decode('utf-8')
1370 # Process video information
1371 self._downloader.process_info({
1372 'id': video_id.decode('utf-8'),
1373 'url': video_url.decode('utf-8'),
1374 'uploader': video_uploader,
1375 'title': video_title,
1376 'stitle': simple_title,
1377 'ext': video_extension.decode('utf-8'),
1381 except UnavailableVideoError:
1382 self._downloader.trouble(u'ERROR: unable to download video')
1385 class YahooIE(InfoExtractor):
1386 """Information extractor for video.yahoo.com."""
1388 # _VALID_URL matches all Yahoo! Video URLs
1389 # _VPAGE_URL matches only the extractable '/watch/' URLs
1390 _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1391 _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1393 def __init__(self, downloader=None):
1394 InfoExtractor.__init__(self, downloader)
1398 return (re.match(YahooIE._VALID_URL, url) is not None)
1400 def report_download_webpage(self, video_id):
1401 """Report webpage download."""
1402 self._downloader.to_stdout(u'[video.yahoo] %s: Downloading webpage' % video_id)
1404 def report_extraction(self, video_id):
1405 """Report information extraction."""
1406 self._downloader.to_stdout(u'[video.yahoo] %s: Extracting information' % video_id)
1408 def _real_initialize(self):
1411 def _real_extract(self, url, new_video=True):
1412 # Extract ID from URL
1413 mobj = re.match(self._VALID_URL, url)
1415 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1418 # At this point we have a new video
1419 self._downloader.increment_downloads()
1420 video_id = mobj.group(2)
1421 video_extension = 'flv'
1423 # Rewrite valid but non-extractable URLs as
1424 # extractable English language /watch/ URLs
1425 if re.match(self._VPAGE_URL, url) is None:
1426 request = urllib2.Request(url)
1428 webpage = urllib2.urlopen(request).read()
1429 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1430 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1433 mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1435 self._downloader.trouble(u'ERROR: Unable to extract id field')
1437 yahoo_id = mobj.group(1)
1439 mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1441 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1443 yahoo_vid = mobj.group(1)
1445 url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1446 return self._real_extract(url, new_video=False)
1448 # Retrieve video webpage to extract further information
1449 request = urllib2.Request(url)
1451 self.report_download_webpage(video_id)
1452 webpage = urllib2.urlopen(request).read()
1453 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1454 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1457 # Extract uploader and title from webpage
1458 self.report_extraction(video_id)
1459 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1461 self._downloader.trouble(u'ERROR: unable to extract video title')
1463 video_title = mobj.group(1).decode('utf-8')
1464 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1466 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1468 self._downloader.trouble(u'ERROR: unable to extract video uploader')
1470 video_uploader = mobj.group(1).decode('utf-8')
1472 # Extract video thumbnail
1473 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1475 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1477 video_thumbnail = mobj.group(1).decode('utf-8')
1479 # Extract video description
1480 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1482 self._downloader.trouble(u'ERROR: unable to extract video description')
1484 video_description = mobj.group(1).decode('utf-8')
1485 if not video_description: video_description = 'No description available.'
1487 # Extract video height and width
1488 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1490 self._downloader.trouble(u'ERROR: unable to extract video height')
1492 yv_video_height = mobj.group(1)
1494 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1496 self._downloader.trouble(u'ERROR: unable to extract video width')
1498 yv_video_width = mobj.group(1)
1500 # Retrieve video playlist to extract media URL
1501 # I'm not completely sure what all these options are, but we
1502 # seem to need most of them, otherwise the server sends a 401.
1503 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
1504 yv_bitrate = '700' # according to Wikipedia this is hard-coded
1505 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1506 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1507 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1509 self.report_download_webpage(video_id)
1510 webpage = urllib2.urlopen(request).read()
1511 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1512 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1515 # Extract media URL from playlist XML
1516 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1518 self._downloader.trouble(u'ERROR: Unable to extract media URL')
1520 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1521 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1524 # Process video information
1525 self._downloader.process_info({
1526 'id': video_id.decode('utf-8'),
1528 'uploader': video_uploader,
1529 'title': video_title,
1530 'stitle': simple_title,
1531 'ext': video_extension.decode('utf-8'),
1532 'thumbnail': video_thumbnail.decode('utf-8'),
1533 'description': video_description,
1534 'thumbnail': video_thumbnail,
1535 'description': video_description,
1538 except UnavailableVideoError:
1539 self._downloader.trouble(u'ERROR: unable to download video')
1542 class GenericIE(InfoExtractor):
1543 """Generic last-resort information extractor."""
1545 def __init__(self, downloader=None):
1546 InfoExtractor.__init__(self, downloader)
1552 def report_download_webpage(self, video_id):
1553 """Report webpage download."""
1554 self._downloader.to_stdout(u'WARNING: Falling back on generic information extractor.')
1555 self._downloader.to_stdout(u'[generic] %s: Downloading webpage' % video_id)
1557 def report_extraction(self, video_id):
1558 """Report information extraction."""
1559 self._downloader.to_stdout(u'[generic] %s: Extracting information' % video_id)
1561 def _real_initialize(self):
1564 def _real_extract(self, url):
1565 # At this point we have a new video
1566 self._downloader.increment_downloads()
1568 video_id = url.split('/')[-1]
1569 request = urllib2.Request(url)
1571 self.report_download_webpage(video_id)
1572 webpage = urllib2.urlopen(request).read()
1573 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1574 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1576 except ValueError, err:
1577 # since this is the last-resort InfoExtractor, if
1578 # this error is thrown, it'll be thrown here
1579 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1582 # Start with something easy: JW Player in SWFObject
1583 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1585 # Broaden the search a little bit
1586 mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1588 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1591 # It's possible that one of the regexes
1592 # matched, but returned an empty group:
1593 if mobj.group(1) is None:
1594 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1597 video_url = urllib.unquote(mobj.group(1))
1598 video_id = os.path.basename(video_url)
1600 # here's a fun little line of code for you:
1601 video_extension = os.path.splitext(video_id)[1][1:]
1602 video_id = os.path.splitext(video_id)[0]
1604 # it's tempting to parse this further, but you would
1605 # have to take into account all the variations like
1606 # Video Title - Site Name
1607 # Site Name | Video Title
1608 # Video Title - Tagline | Site Name
1609 # and so on and so forth; it's just not practical
1610 mobj = re.search(r'<title>(.*)</title>', webpage)
1612 self._downloader.trouble(u'ERROR: unable to extract title')
1614 video_title = mobj.group(1).decode('utf-8')
1615 video_title = sanitize_title(video_title)
1616 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1618 # video uploader is domain name
1619 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1621 self._downloader.trouble(u'ERROR: unable to extract title')
1623 video_uploader = mobj.group(1).decode('utf-8')
1626 # Process video information
1627 self._downloader.process_info({
1628 'id': video_id.decode('utf-8'),
1629 'url': video_url.decode('utf-8'),
1630 'uploader': video_uploader,
1631 'title': video_title,
1632 'stitle': simple_title,
1633 'ext': video_extension.decode('utf-8'),
1637 except UnavailableVideoError, err:
1638 self._downloader.trouble(u'ERROR: unable to download video')
1641 class YoutubeSearchIE(InfoExtractor):
1642 """Information Extractor for YouTube search queries."""
1643 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1644 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1645 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1646 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1648 _max_youtube_results = 1000
1650 def __init__(self, youtube_ie, downloader=None):
1651 InfoExtractor.__init__(self, downloader)
1652 self._youtube_ie = youtube_ie
1656 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1658 def report_download_page(self, query, pagenum):
1659 """Report attempt to download playlist page with given number."""
1660 query = query.decode(preferredencoding())
1661 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1663 def _real_initialize(self):
1664 self._youtube_ie.initialize()
1666 def _real_extract(self, query):
1667 mobj = re.match(self._VALID_QUERY, query)
1669 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1672 prefix, query = query.split(':')
1674 query = query.encode('utf-8')
1676 self._download_n_results(query, 1)
1678 elif prefix == 'all':
1679 self._download_n_results(query, self._max_youtube_results)
1685 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1687 elif n > self._max_youtube_results:
1688 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1689 n = self._max_youtube_results
1690 self._download_n_results(query, n)
1692 except ValueError: # parsing prefix as integer fails
1693 self._download_n_results(query, 1)
1696 def _download_n_results(self, query, n):
1697 """Downloads a specified number of results for a query"""
1700 already_seen = set()
1704 self.report_download_page(query, pagenum)
1705 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1706 request = urllib2.Request(result_url, None, std_headers)
1708 page = urllib2.urlopen(request).read()
1709 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1710 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1713 # Extract video identifiers
1714 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1715 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1716 if video_id not in already_seen:
1717 video_ids.append(video_id)
1718 already_seen.add(video_id)
1719 if len(video_ids) == n:
1720 # Specified n videos reached
1721 for id in video_ids:
1722 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1725 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1726 for id in video_ids:
1727 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1730 pagenum = pagenum + 1
1732 class GoogleSearchIE(InfoExtractor):
1733 """Information Extractor for Google Video search queries."""
1734 _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
1735 _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1736 _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
1737 _MORE_PAGES_INDICATOR = r'<span>Next</span>'
1739 _max_google_results = 1000
1741 def __init__(self, google_ie, downloader=None):
1742 InfoExtractor.__init__(self, downloader)
1743 self._google_ie = google_ie
1747 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
1749 def report_download_page(self, query, pagenum):
1750 """Report attempt to download playlist page with given number."""
1751 query = query.decode(preferredencoding())
1752 self._downloader.to_stdout(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1754 def _real_initialize(self):
1755 self._google_ie.initialize()
1757 def _real_extract(self, query):
1758 mobj = re.match(self._VALID_QUERY, query)
1760 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1763 prefix, query = query.split(':')
1765 query = query.encode('utf-8')
1767 self._download_n_results(query, 1)
1769 elif prefix == 'all':
1770 self._download_n_results(query, self._max_google_results)
1776 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1778 elif n > self._max_google_results:
1779 self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1780 n = self._max_google_results
1781 self._download_n_results(query, n)
1783 except ValueError: # parsing prefix as integer fails
1784 self._download_n_results(query, 1)
1787 def _download_n_results(self, query, n):
1788 """Downloads a specified number of results for a query"""
1791 already_seen = set()
1795 self.report_download_page(query, pagenum)
1796 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1797 request = urllib2.Request(result_url, None, std_headers)
1799 page = urllib2.urlopen(request).read()
1800 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1801 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1804 # Extract video identifiers
1805 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1806 video_id = mobj.group(1)
1807 if video_id not in already_seen:
1808 video_ids.append(video_id)
1809 already_seen.add(video_id)
1810 if len(video_ids) == n:
1811 # Specified n videos reached
1812 for id in video_ids:
1813 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1816 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1817 for id in video_ids:
1818 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1821 pagenum = pagenum + 1
1823 class YahooSearchIE(InfoExtractor):
1824 """Information Extractor for Yahoo! Video search queries."""
1825 _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
1826 _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1827 _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1828 _MORE_PAGES_INDICATOR = r'\s*Next'
1830 _max_yahoo_results = 1000
1832 def __init__(self, yahoo_ie, downloader=None):
1833 InfoExtractor.__init__(self, downloader)
1834 self._yahoo_ie = yahoo_ie
1838 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
1840 def report_download_page(self, query, pagenum):
1841 """Report attempt to download playlist page with given number."""
1842 query = query.decode(preferredencoding())
1843 self._downloader.to_stdout(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1845 def _real_initialize(self):
1846 self._yahoo_ie.initialize()
1848 def _real_extract(self, query):
1849 mobj = re.match(self._VALID_QUERY, query)
1851 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1854 prefix, query = query.split(':')
1856 query = query.encode('utf-8')
1858 self._download_n_results(query, 1)
1860 elif prefix == 'all':
1861 self._download_n_results(query, self._max_yahoo_results)
1867 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1869 elif n > self._max_yahoo_results:
1870 self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1871 n = self._max_yahoo_results
1872 self._download_n_results(query, n)
1874 except ValueError: # parsing prefix as integer fails
1875 self._download_n_results(query, 1)
1878 def _download_n_results(self, query, n):
1879 """Downloads a specified number of results for a query"""
1882 already_seen = set()
1886 self.report_download_page(query, pagenum)
1887 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1888 request = urllib2.Request(result_url, None, std_headers)
1890 page = urllib2.urlopen(request).read()
1891 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1892 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1895 # Extract video identifiers
1896 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1897 video_id = mobj.group(1)
1898 if video_id not in already_seen:
1899 video_ids.append(video_id)
1900 already_seen.add(video_id)
1901 if len(video_ids) == n:
1902 # Specified n videos reached
1903 for id in video_ids:
1904 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1907 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1908 for id in video_ids:
1909 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1912 pagenum = pagenum + 1
1914 class YoutubePlaylistIE(InfoExtractor):
1915 """Information Extractor for YouTube playlists."""
1917 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
1918 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1919 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1920 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1923 def __init__(self, youtube_ie, downloader=None):
1924 InfoExtractor.__init__(self, downloader)
1925 self._youtube_ie = youtube_ie
1929 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1931 def report_download_page(self, playlist_id, pagenum):
1932 """Report attempt to download playlist page with given number."""
1933 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1935 def _real_initialize(self):
1936 self._youtube_ie.initialize()
1938 def _real_extract(self, url):
1939 # Extract playlist id
1940 mobj = re.match(self._VALID_URL, url)
1942 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1945 # Download playlist pages
1946 playlist_id = mobj.group(1)
1951 self.report_download_page(playlist_id, pagenum)
1952 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1954 page = urllib2.urlopen(request).read()
1955 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1956 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1959 # Extract video identifiers
1961 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1962 if mobj.group(1) not in ids_in_page:
1963 ids_in_page.append(mobj.group(1))
1964 video_ids.extend(ids_in_page)
1966 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1968 pagenum = pagenum + 1
1970 for id in video_ids:
1971 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1974 class YoutubeUserIE(InfoExtractor):
1975 """Information Extractor for YouTube users."""
1977 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
1978 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1979 _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
1982 def __init__(self, youtube_ie, downloader=None):
1983 InfoExtractor.__init__(self, downloader)
1984 self._youtube_ie = youtube_ie
1988 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
1990 def report_download_page(self, username):
1991 """Report attempt to download user page."""
1992 self._downloader.to_stdout(u'[youtube] user %s: Downloading page ' % (username))
1994 def _real_initialize(self):
1995 self._youtube_ie.initialize()
1997 def _real_extract(self, url):
1999 mobj = re.match(self._VALID_URL, url)
2001 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2004 # Download user page
2005 username = mobj.group(1)
2009 self.report_download_page(username)
2010 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
2012 page = urllib2.urlopen(request).read()
2013 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2014 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2017 # Extract video identifiers
2020 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2021 if mobj.group(1) not in ids_in_page:
2022 ids_in_page.append(mobj.group(1))
2023 video_ids.extend(ids_in_page)
2025 for id in video_ids:
2026 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2029 class PostProcessor(object):
2030 """Post Processor class.
2032 PostProcessor objects can be added to downloaders with their
2033 add_post_processor() method. When the downloader has finished a
2034 successful download, it will take its internal chain of PostProcessors
2035 and start calling the run() method on each one of them, first with
2036 an initial argument and then with the returned value of the previous
2039 The chain will be stopped if one of them ever returns None or the end
2040 of the chain is reached.
2042 PostProcessor objects follow a "mutual registration" process similar
2043 to InfoExtractor objects.
2048 def __init__(self, downloader=None):
2049 self._downloader = downloader
2051 def set_downloader(self, downloader):
2052 """Sets the downloader for this PP."""
2053 self._downloader = downloader
2055 def run(self, information):
2056 """Run the PostProcessor.
2058 The "information" argument is a dictionary like the ones
2059 composed by InfoExtractors. The only difference is that this
2060 one has an extra field called "filepath" that points to the
2063 When this method returns None, the postprocessing chain is
2064 stopped. However, this method may return an information
2065 dictionary that will be passed to the next postprocessing
2066 object in the chain. It can be the one it received after
2067 changing some fields.
2069 In addition, this method may raise a PostProcessingError
2070 exception that will be taken into account by the downloader
2073 return information # by default, do nothing
2075 ### MAIN PROGRAM ###
2076 if __name__ == '__main__':
2078 # Modules needed only when running the main program
2082 # Function to update the program file with the latest version from bitbucket.org
2083 def update_self(downloader, filename):
2084 # Note: downloader only used for options
2085 if not os.access (filename, os.W_OK):
2086 sys.exit('ERROR: no write permissions on %s' % filename)
2088 downloader.to_stdout('Updating to latest stable version...')
2089 latest_url = 'http://bitbucket.org/rg3/youtube-dl/raw/tip/LATEST_VERSION'
2090 latest_version = urllib.urlopen(latest_url).read().strip()
2091 prog_url = 'http://bitbucket.org/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
2092 newcontent = urllib.urlopen(prog_url).read()
2093 stream = open(filename, 'w')
2094 stream.write(newcontent)
2096 downloader.to_stdout('Updated to version %s' % latest_version)
2098 # General configuration
2099 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
2100 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
2101 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
2103 # Parse command line
2104 parser = optparse.OptionParser(
2105 usage='Usage: %prog [options] url...',
2106 version='2010.07.14',
2107 conflict_handler='resolve',
2110 parser.add_option('-h', '--help',
2111 action='help', help='print this help text and exit')
2112 parser.add_option('-v', '--version',
2113 action='version', help='print program version and exit')
2114 parser.add_option('-U', '--update',
2115 action='store_true', dest='update_self', help='update this program to latest stable version')
2116 parser.add_option('-i', '--ignore-errors',
2117 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
2118 parser.add_option('-r', '--rate-limit',
2119 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
2120 parser.add_option('-R', '--retries',
2121 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
2123 authentication = optparse.OptionGroup(parser, 'Authentication Options')
2124 authentication.add_option('-u', '--username',
2125 dest='username', metavar='USERNAME', help='account username')
2126 authentication.add_option('-p', '--password',
2127 dest='password', metavar='PASSWORD', help='account password')
2128 authentication.add_option('-n', '--netrc',
2129 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
2130 parser.add_option_group(authentication)
2132 video_format = optparse.OptionGroup(parser, 'Video Format Options')
2133 video_format.add_option('-f', '--format',
2134 action='store', dest='format', metavar='FORMAT', help='video format code')
2135 video_format.add_option('-b', '--best-quality',
2136 action='store_const', dest='format', help='download the best quality video possible', const='0')
2137 video_format.add_option('-m', '--mobile-version',
2138 action='store_const', dest='format', help='alias for -f 17', const='17')
2139 video_format.add_option('-d', '--high-def',
2140 action='store_const', dest='format', help='alias for -f 22', const='22')
2141 video_format.add_option('--all-formats',
2142 action='store_const', dest='format', help='download all available video formats', const='-1')
2143 video_format.add_option('--max-quality',
2144 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format limit for -b')
2145 parser.add_option_group(video_format)
2147 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
2148 verbosity.add_option('-q', '--quiet',
2149 action='store_true', dest='quiet', help='activates quiet mode', default=False)
2150 verbosity.add_option('-s', '--simulate',
2151 action='store_true', dest='simulate', help='do not download video', default=False)
2152 verbosity.add_option('-g', '--get-url',
2153 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
2154 verbosity.add_option('-e', '--get-title',
2155 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
2156 verbosity.add_option('--get-thumbnail',
2157 action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False)
2158 verbosity.add_option('--get-description',
2159 action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
2160 verbosity.add_option('--no-progress',
2161 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
2162 parser.add_option_group(verbosity)
2164 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
2165 filesystem.add_option('-t', '--title',
2166 action='store_true', dest='usetitle', help='use title in file name', default=False)
2167 filesystem.add_option('-l', '--literal',
2168 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
2169 filesystem.add_option('-o', '--output',
2170 dest='outtmpl', metavar='TEMPLATE', help='output filename template')
2171 filesystem.add_option('-a', '--batch-file',
2172 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
2173 filesystem.add_option('-w', '--no-overwrites',
2174 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
2175 filesystem.add_option('-c', '--continue',
2176 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
2177 parser.add_option_group(filesystem)
2179 (opts, args) = parser.parse_args()
2181 # Batch file verification
2183 if opts.batchfile is not None:
2185 if opts.batchfile == '-':
2188 batchfd = open(opts.batchfile, 'r')
2189 batchurls = batchfd.readlines()
2190 batchurls = [x.strip() for x in batchurls]
2191 batchurls = [x for x in batchurls if len(x) > 0]
2193 sys.exit(u'ERROR: batch file could not be read')
2194 all_urls = batchurls + args
2196 # Conflicting, missing and erroneous options
2197 if opts.usenetrc and (opts.username is not None or opts.password is not None):
2198 parser.error(u'using .netrc conflicts with giving username/password')
2199 if opts.password is not None and opts.username is None:
2200 parser.error(u'account username missing')
2201 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
2202 parser.error(u'using output template conflicts with using title or literal title')
2203 if opts.usetitle and opts.useliteral:
2204 parser.error(u'using title conflicts with using literal title')
2205 if opts.username is not None and opts.password is None:
2206 opts.password = getpass.getpass(u'Type account password and press return:')
2207 if opts.ratelimit is not None:
2208 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
2209 if numeric_limit is None:
2210 parser.error(u'invalid rate limit specified')
2211 opts.ratelimit = numeric_limit
2212 if opts.retries is not None:
2214 opts.retries = long(opts.retries)
2215 except (TypeError, ValueError), err:
2216 parser.error(u'invalid retry count specified')
2218 # Information extractors
2219 youtube_ie = YoutubeIE()
2220 metacafe_ie = MetacafeIE(youtube_ie)
2221 dailymotion_ie = DailymotionIE()
2222 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
2223 youtube_user_ie = YoutubeUserIE(youtube_ie)
2224 youtube_search_ie = YoutubeSearchIE(youtube_ie)
2225 google_ie = GoogleIE()
2226 google_search_ie = GoogleSearchIE(google_ie)
2227 photobucket_ie = PhotobucketIE()
2228 yahoo_ie = YahooIE()
2229 yahoo_search_ie = YahooSearchIE(yahoo_ie)
2230 generic_ie = GenericIE()
2233 fd = FileDownloader({
2234 'usenetrc': opts.usenetrc,
2235 'username': opts.username,
2236 'password': opts.password,
2237 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2238 'forceurl': opts.geturl,
2239 'forcetitle': opts.gettitle,
2240 'forcethumbnail': opts.getthumbnail,
2241 'forcedescription': opts.getdescription,
2242 'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2243 'format': opts.format,
2244 'format_limit': opts.format_limit,
2245 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
2246 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
2247 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
2248 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
2249 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
2250 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
2251 or u'%(id)s.%(ext)s'),
2252 'ignoreerrors': opts.ignoreerrors,
2253 'ratelimit': opts.ratelimit,
2254 'nooverwrites': opts.nooverwrites,
2255 'retries': opts.retries,
2256 'continuedl': opts.continue_dl,
2257 'noprogress': opts.noprogress,
2259 fd.add_info_extractor(youtube_search_ie)
2260 fd.add_info_extractor(youtube_pl_ie)
2261 fd.add_info_extractor(youtube_user_ie)
2262 fd.add_info_extractor(metacafe_ie)
2263 fd.add_info_extractor(dailymotion_ie)
2264 fd.add_info_extractor(youtube_ie)
2265 fd.add_info_extractor(google_ie)
2266 fd.add_info_extractor(google_search_ie)
2267 fd.add_info_extractor(photobucket_ie)
2268 fd.add_info_extractor(yahoo_ie)
2269 fd.add_info_extractor(yahoo_search_ie)
2271 # This must come last since it's the
2272 # fallback if none of the others work
2273 fd.add_info_extractor(generic_ie)
2276 if opts.update_self:
2277 update_self(fd, sys.argv[0])
2280 if len(all_urls) < 1:
2281 if not opts.update_self:
2282 parser.error(u'you must provide at least one URL')
2285 retcode = fd.download(all_urls)
2288 except DownloadError:
2290 except SameFileError:
2291 sys.exit(u'ERROR: fixed output name but more than one file to download')
2292 except KeyboardInterrupt:
2293 sys.exit(u'\nERROR: Interrupted by user')