Remove calls to _downloader.download in Youtube searchs

[youtube-dl] / youtube_dl / FileDownloader.py

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index d2b9be9ef88cb93e3e54fb0d3650880e35c36b43..9c0c42f8d44ab5a85df6fd68f272869ca546f940 100644 (file)
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -17,6 +17,7 @@ if os.name == 'nt':
     import ctypes
 
 from .utils import *
     import ctypes
 
 from .utils import *

+from .InfoExtractors import get_info_extractor

 
 
 class FileDownloader(object):
 
 
 class FileDownloader(object):


@@ -231,11 +232,21 @@ class FileDownloader(object):
             self.to_stderr(message)
         if self.params.get('verbose'):
             if tb is None:
             self.to_stderr(message)
         if self.params.get('verbose'):
             if tb is None:

-                tb_data = traceback.format_list(traceback.extract_stack())
-                tb = u''.join(tb_data)
+                if sys.exc_info()[0]:  # if .trouble has been called from an except block
+                    tb = u''
+                    if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                        tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
+                    tb += compat_str(traceback.format_exc())
+                else:
+                    tb_data = traceback.format_list(traceback.extract_stack())
+                    tb = u''.join(tb_data)

             self.to_stderr(tb)
         if not self.params.get('ignoreerrors', False):
             self.to_stderr(tb)
         if not self.params.get('ignoreerrors', False):

-            raise DownloadError(message)
+            if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                exc_info = sys.exc_info()[1].exc_info
+            else:
+                exc_info = sys.exc_info()
+            raise DownloadError(message, exc_info)

         self._download_retcode = 1
 
     def report_warning(self, message):
         self._download_retcode = 1
 
     def report_warning(self, message):


@@ -250,6 +261,18 @@ class FileDownloader(object):
         warning_message=u'%s %s' % (_msg_header,message)
         self.to_stderr(warning_message)
 
         warning_message=u'%s %s' % (_msg_header,message)
         self.to_stderr(warning_message)
 

+    def report_error(self, message, tb=None):
+        '''
+        Do the same as trouble, but prefixes the message with 'ERROR:', colored
+        in red if stderr is a tty file.
+        '''
+        if sys.stderr.isatty():
+            _msg_header = u'\033[0;31mERROR:\033[0m'
+        else:
+            _msg_header = u'ERROR:'
+        error_message = u'%s %s' % (_msg_header, message)
+        self.trouble(error_message, tb)
+

     def slow_down(self, start_time, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
     def slow_down(self, start_time, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)


@@ -281,7 +304,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:

-            self.trouble(u'ERROR: unable to rename file')
+            self.report_error(u'unable to rename file')

 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""


@@ -366,7 +389,13 @@ class FileDownloader(object):
             template_dict = dict(info_dict)
 
             template_dict['epoch'] = int(time.time())
             template_dict = dict(info_dict)
 
             template_dict['epoch'] = int(time.time())

-            template_dict['autonumber'] = u'%05d' % self._num_downloads
+            autonumber_size = self.params.get('autonumber_size')
+            if autonumber_size is None:
+                autonumber_size = 5
+            autonumber_templ = u'%0' + str(autonumber_size) + u'd'
+            template_dict['autonumber'] = autonumber_templ % self._num_downloads
+            if template_dict['playlist_index'] is not None:
+                template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']

 
             sanitize = lambda k,v: sanitize_filename(
                 u'NA' if v is None else compat_str(v),
 
             sanitize = lambda k,v: sanitize_filename(
                 u'NA' if v is None else compat_str(v),


@@ -397,12 +426,23 @@ class FileDownloader(object):
                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
         return None
         
                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
         return None
         

-    def extract_info(self, url):
+    def extract_info(self, url, download = True, ie_name = None):

         '''
         Returns a list with a dictionary for each video we find.
         '''
         Returns a list with a dictionary for each video we find.

+        If 'download', also downloads the videos.

          '''
         suitable_found = False
          '''
         suitable_found = False

-        for ie in self._ies:
+        
+        #We copy the original list
+        ies = list(self._ies)
+
+        if ie_name is not None:
+            #We put in the first place the given info extractor
+            first_ie = get_info_extractor(ie_name)()
+            first_ie.set_downloader(self)
+            ies.insert(0, first_ie)
+
+        for ie in ies:

             # Go to next InfoExtractor if not suitable
             if not ie.suitable(url):
                 continue
             # Go to next InfoExtractor if not suitable
             if not ie.suitable(url):
                 continue


@@ -418,7 +458,12 @@ class FileDownloader(object):
             # Extract information from URL and process it
             try:
                 ie_results = ie.extract(url)
             # Extract information from URL and process it
             try:
                 ie_results = ie.extract(url)

-                results = self.process_ie_results(ie_results, ie)
+                results = []
+                for ie_result in ie_results:
+                    if not 'extractor' in ie_result:
+                        #The extractor has already been set somewhere else
+                        ie_result['extractor'] = ie.IE_NAME
+                    results.append(self.process_ie_result(ie_result, download))

                 return results
             except ExtractorError as de: # An error we somewhat expected
                 self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
                 return results
             except ExtractorError as de: # An error we somewhat expected
                 self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())


@@ -431,50 +476,73 @@ class FileDownloader(object):
                     raise
         if not suitable_found:
                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
                     raise
         if not suitable_found:
                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)

-    def extract_info_iterable(self, urls):
-        '''
-            Return the videos founded for the urls
-        '''
-        results = []
-        for url in urls:
-            results.extend(self.extract_info(url))
-        return results

         
         

-    def process_ie_results(self, ie_results, ie):
+    def process_ie_result(self, ie_result, download = True):

         """
         """

-        Take the results of the ie and return a list of videos.
-        For url elements it will seartch the suitable ie and get the videos
+        Take the result of the ie and return a list of videos.
+        For url elements it will search the suitable ie and get the videos

         For playlist elements it will process each of the elements of the 'entries' key
         For playlist elements it will process each of the elements of the 'entries' key

+        
+        It will also download the videos if 'download'.

         """
         """

-        results = [] 
-        for result in ie_results or []:
-            result_type = result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system
-            if result_type == 'video':
-                if not 'extractor' in result:
-                    #The extractor has already been set somewhere else
-                    result['extractor'] = ie.IE_NAME
-                results.append(result)
-            elif result_type == 'url':
-                #We get the videos pointed by the url
-                results.extend(self.extract_info(result['url']))
-            elif result_type == 'playlist':
-                #We process each entry in the playlist
-                entries_result = self.process_ie_results(result['entries'], ie)
-                result['entries'] = entries_result
-                results.extend([result])
-        return results
+        result_type = ie_result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system
+        if result_type == 'video':
+            if 'playlist' not in ie_result:
+                #It isn't part of a playlist
+                ie_result['playlist'] = None
+                ie_result['playlist_index'] = None
+            if download:
+                #Do the download:
+                self.process_info(ie_result)
+            return ie_result
+        elif result_type == 'url':
+            #We get the video pointed by the url
+            result = self.extract_info(ie_result['url'], download, ie_name = ie_result['ie_key'])[0]
+            return result
+        elif result_type == 'playlist':
+            #We process each entry in the playlist
+            playlist = ie_result.get('title', None) or ie_result.get('id', None)
+            self.to_screen(u'[download] Downloading playlist: %s'  % playlist)
+
+            playlist_results = []
+
+            n_all_entries = len(ie_result['entries'])
+            playliststart = self.params.get('playliststart', 1) - 1
+            playlistend = self.params.get('playlistend', -1)
+
+            if playlistend == -1:
+                entries = ie_result['entries'][playliststart:]
+            else:
+                entries = ie_result['entries'][playliststart:playlistend]
+
+            n_entries = len(entries)
+
+            self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
+                (ie_result['extractor'], playlist, n_all_entries, n_entries))
+
+            for i,entry in enumerate(entries,1):
+                self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
+                entry_result = self.process_ie_result(entry, False)
+                entry_result['playlist'] = playlist
+                entry_result['playlist_index'] = i + playliststart
+                #We must do the download here to correctly set the 'playlist' key
+                if download:
+                    self.process_info(entry_result)
+                playlist_results.append(entry_result)
+            result = ie_result.copy()
+            result['entries'] = playlist_results
+            return result

 
     def process_info(self, info_dict):
         """Process a single dictionary returned by an InfoExtractor."""
 
 
     def process_info(self, info_dict):
         """Process a single dictionary returned by an InfoExtractor."""
 

-        if info_dict.get('_type','video') == 'playlist':
-            playlist = info_dict.get('title', None) or info_dict.get('id', None)
-            self.to_screen(u'[download] Downloading playlist: %s'  % playlist)
-            for video in info_dict['entries']:
-                video['playlist'] = playlist
-                self.process_info(video)
-            return
+        #We increment the download the download count here to match the previous behaviour.
+        self.increment_downloads()

         
         

+        info_dict['fulltitle'] = info_dict['title']
+        if len(info_dict['title']) > 200:
+            info_dict['title'] = info_dict['title'][:197] + u'...'
+

         # Keep for backwards compatibility
         info_dict['stitle'] = info_dict['title']
 
         # Keep for backwards compatibility
         info_dict['stitle'] = info_dict['title']
 


@@ -519,7 +587,7 @@ class FileDownloader(object):
             if dn != '' and not os.path.exists(dn): # dn is already encoded
                 os.makedirs(dn)
         except (OSError, IOError) as err:
             if dn != '' and not os.path.exists(dn): # dn is already encoded
                 os.makedirs(dn)
         except (OSError, IOError) as err:

-            self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
+            self.report_error(u'unable to create directory ' + compat_str(err))

             return
 
         if self.params.get('writedescription', False):
             return
 
         if self.params.get('writedescription', False):


@@ -529,7 +597,7 @@ class FileDownloader(object):
                 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                     descfile.write(info_dict['description'])
             except (OSError, IOError):
                 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                     descfile.write(info_dict['description'])
             except (OSError, IOError):

-                self.trouble(u'ERROR: Cannot write description file ' + descfn)
+                self.report_error(u'Cannot write description file ' + descfn)

                 return
 
         if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
                 return
 
         if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:


@@ -538,14 +606,17 @@ class FileDownloader(object):
             subtitle = info_dict['subtitles'][0]
             (sub_error, sub_lang, sub) = subtitle
             sub_format = self.params.get('subtitlesformat')
             subtitle = info_dict['subtitles'][0]
             (sub_error, sub_lang, sub) = subtitle
             sub_format = self.params.get('subtitlesformat')

-            try:
-                sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
-                self.report_writesubtitles(sub_filename)
-                with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
-                    subfile.write(sub)
-            except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
-                return
+            if sub_error:
+                self.report_warning("Some error while getting the subtitles")
+            else:
+                try:
+                    sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+                    self.report_writesubtitles(sub_filename)
+                    with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+                        subfile.write(sub)
+                except (OSError, IOError):
+                    self.report_error(u'Cannot write subtitles file ' + descfn)
+                    return

             if self.params.get('onlysubtitles', False):
                 return 
 
             if self.params.get('onlysubtitles', False):
                 return 
 


@@ -554,14 +625,17 @@ class FileDownloader(object):
             sub_format = self.params.get('subtitlesformat')
             for subtitle in subtitles:
                 (sub_error, sub_lang, sub) = subtitle
             sub_format = self.params.get('subtitlesformat')
             for subtitle in subtitles:
                 (sub_error, sub_lang, sub) = subtitle

-                try:
-                    sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
-                    self.report_writesubtitles(sub_filename)
-                    with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
-                            subfile.write(sub)
-                except (OSError, IOError):
-                    self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
-                    return
+                if sub_error:
+                    self.report_warning("Some error while getting the subtitles")
+                else:
+                    try:
+                        sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+                        self.report_writesubtitles(sub_filename)
+                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+                                subfile.write(sub)
+                    except (OSError, IOError):
+                        self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+                        return

             if self.params.get('onlysubtitles', False):
                 return 
 
             if self.params.get('onlysubtitles', False):
                 return 
 


@@ -572,7 +646,7 @@ class FileDownloader(object):
                 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
                 write_json_file(json_info_dict, encodeFilename(infofn))
             except (OSError, IOError):
                 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
                 write_json_file(json_info_dict, encodeFilename(infofn))
             except (OSError, IOError):

-                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
+                self.report_error(u'Cannot write metadata to JSON file ' + infofn)

                 return
 
         if not self.params.get('skip_download', False):
                 return
 
         if not self.params.get('skip_download', False):


@@ -584,17 +658,17 @@ class FileDownloader(object):
                 except (OSError, IOError) as err:
                     raise UnavailableVideoError()
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                 except (OSError, IOError) as err:
                     raise UnavailableVideoError()
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:

-                    self.trouble(u'ERROR: unable to download video data: %s' % str(err))
+                    self.report_error(u'unable to download video data: %s' % str(err))

                     return
                 except (ContentTooShortError, ) as err:
                     return
                 except (ContentTooShortError, ) as err:

-                    self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                    self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))

                     return
 
             if success:
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
                     return
 
             if success:
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:

-                    self.trouble(u'ERROR: postprocessing: %s' % str(err))
+                    self.report_error(u'postprocessing: %s' % str(err))

                     return
 
     def download(self, url_list):
                     return
 
     def download(self, url_list):


@@ -603,14 +677,14 @@ class FileDownloader(object):
             raise SameFileError(self.params['outtmpl'])
 
         for url in url_list:
             raise SameFileError(self.params['outtmpl'])
 
         for url in url_list:

-            videos = self.extract_info(url)
-
-            for video in videos or []:
-                try:
-                    self.increment_downloads()
-                    self.process_info(video)
-                except UnavailableVideoError:
-                    self.trouble(u'\nERROR: unable to download video')
+            try:
+                #It also downloads the videos
+                videos = self.extract_info(url)
+            except UnavailableVideoError:
+                self.trouble(u'\nERROR: unable to download video')
+            except MaxDownloadsReached:
+                self.to_screen(u'[info] Maximum number of downloaded files reached.')
+                raise

 
         return self._download_retcode
 
 
         return self._download_retcode
 


@@ -637,7 +711,7 @@ class FileDownloader(object):
             except (IOError, OSError):
                 self.report_warning(u'Unable to remove downloaded video file')
 
             except (IOError, OSError):
                 self.report_warning(u'Unable to remove downloaded video file')
 

-    def _download_with_rtmpdump(self, filename, url, player_url, page_url):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path):

         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 


@@ -645,7 +719,7 @@ class FileDownloader(object):
         try:
             subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
         except (OSError, IOError):
         try:
             subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
         except (OSError, IOError):

-            self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
+            self.report_error(u'RTMP download detected but "rtmpdump" could not be run')

             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when


@@ -656,6 +730,8 @@ class FileDownloader(object):
             basic_args += ['-W', player_url]
         if page_url is not None:
             basic_args += ['--pageUrl', page_url]
             basic_args += ['-W', player_url]
         if page_url is not None:
             basic_args += ['--pageUrl', page_url]

+        if play_path is not None:
+            basic_args += ['-y', play_path]

         args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
         args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:


@@ -690,7 +766,8 @@ class FileDownloader(object):
             })
             return True
         else:
             })
             return True
         else:

-            self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
+            self.to_stderr(u"\n")
+            self.report_error(u'rtmpdump exited with code %d' % retval)

             return False
 
     def _do_download(self, filename, info_dict):
             return False
 
     def _do_download(self, filename, info_dict):


@@ -709,7 +786,8 @@ class FileDownloader(object):
         if url.startswith('rtmp'):
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),
         if url.startswith('rtmp'):
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),

-                                                info_dict.get('page_url', None))
+                                                info_dict.get('page_url', None),
+                                                info_dict.get('play_path', None))

 
         tmpfilename = self.temp_name(filename)
         stream = None
 
         tmpfilename = self.temp_name(filename)
         stream = None


@@ -790,7 +868,7 @@ class FileDownloader(object):
                 self.report_retry(count, retries)
 
         if count > retries:
                 self.report_retry(count, retries)
 
         if count > retries:

-            self.trouble(u'ERROR: giving up after %s retries' % retries)
+            self.report_error(u'giving up after %s retries' % retries)

             return False
 
         data_len = data.info().get('Content-length', None)
             return False
 
         data_len = data.info().get('Content-length', None)


@@ -826,12 +904,13 @@ class FileDownloader(object):
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:

-                    self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
+                    self.report_error(u'unable to open for writing: %s' % str(err))

                     return False
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
                     return False
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:

-                self.trouble(u'\nERROR: unable to write data: %s' % str(err))
+                self.to_stderr(u"\n")
+                self.report_error(u'unable to write data: %s' % str(err))

                 return False
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
                 return False
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))


@@ -857,7 +936,8 @@ class FileDownloader(object):
             self.slow_down(start, byte_counter - resume_len)
 
         if stream is None:
             self.slow_down(start, byte_counter - resume_len)
 
         if stream is None:

-            self.trouble(u'\nERROR: Did not get any data blocks')
+            self.to_stderr(u"\n")
+            self.report_error(u'Did not get any data blocks')

             return False
         stream.close()
         self.report_finish()
             return False
         stream.close()
         self.report_finish()