[Instagram] get the non-https link, as they are serving Akamai cert from a instagram...

[youtube-dl] / youtube_dl / FileDownloader.py

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index ea6b9d626efa7a18eafe20afa8c473d1afee315b..8ecabab1a517467c118dad0857c47291bcb2f929 100644 (file)
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -63,32 +63,57 @@ class FileDownloader(object):
         converted = float(bytes) / float(1024 ** exponent)
         return '%.2f%s' % (converted, suffix)
 
         converted = float(bytes) / float(1024 ** exponent)
         return '%.2f%s' % (converted, suffix)
 

+    @staticmethod
+    def format_seconds(seconds):
+        (mins, secs) = divmod(seconds, 60)
+        (hours, mins) = divmod(mins, 60)
+        if hours > 99:
+            return '--:--:--'
+        if hours == 0:
+            return '%02d:%02d' % (mins, secs)
+        else:
+            return '%02d:%02d:%02d' % (hours, mins, secs)
+

     @staticmethod
     def calc_percent(byte_counter, data_len):
         if data_len is None:
     @staticmethod
     def calc_percent(byte_counter, data_len):
         if data_len is None:

+            return None
+        return float(byte_counter) / float(data_len) * 100.0
+
+    @staticmethod
+    def format_percent(percent):
+        if percent is None:

             return '---.-%'
             return '---.-%'

-        return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
+        return '%6s' % ('%3.1f%%' % percent)

 
     @staticmethod
     def calc_eta(start, now, total, current):
         if total is None:
 
     @staticmethod
     def calc_eta(start, now, total, current):
         if total is None:

-            return '--:--'
+            return None

         dif = now - start
         if current == 0 or dif < 0.001: # One millisecond
         dif = now - start
         if current == 0 or dif < 0.001: # One millisecond

-            return '--:--'
+            return None

         rate = float(current) / dif
         rate = float(current) / dif

-        eta = int((float(total) - float(current)) / rate)
-        (eta_mins, eta_secs) = divmod(eta, 60)
-        if eta_mins > 99:
+        return int((float(total) - float(current)) / rate)
+
+    @staticmethod
+    def format_eta(eta):
+        if eta is None:

             return '--:--'
             return '--:--'

-        return '%02d:%02d' % (eta_mins, eta_secs)
+        return FileDownloader.format_seconds(eta)

 
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
         if bytes == 0 or dif < 0.001: # One millisecond
 
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
         if bytes == 0 or dif < 0.001: # One millisecond

+            return None
+        return float(bytes) / dif
+
+    @staticmethod
+    def format_speed(speed):
+        if speed is None:

             return '%10s' % '---b/s'
             return '%10s' % '---b/s'

-        return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
+        return '%10s' % ('%s/s' % FileDownloader.format_bytes(speed))

 
     @staticmethod
     def best_block_size(elapsed_time, bytes):
 
     @staticmethod
     def best_block_size(elapsed_time, bytes):


@@ -197,11 +222,14 @@ class FileDownloader(object):
         """Report destination filename."""
         self.to_screen(u'[download] Destination: ' + filename)
 
         """Report destination filename."""
         self.to_screen(u'[download] Destination: ' + filename)
 

-    def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
+    def report_progress(self, percent, data_len_str, speed, eta):

         """Report download progress."""
         if self.params.get('noprogress', False):
             return
         clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
         """Report download progress."""
         if self.params.get('noprogress', False):
             return
         clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')

+        eta_str = self.format_eta(eta)
+        percent_str = self.format_percent(percent)
+        speed_str = self.format_speed(speed)

         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str))
         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str))


@@ -230,16 +258,19 @@ class FileDownloader(object):
         """Report it was impossible to resume download."""
         self.to_screen(u'[download] Unable to resume')
 
         """Report it was impossible to resume download."""
         self.to_screen(u'[download] Unable to resume')
 

-    def report_finish(self):
+    def report_finish(self, data_len_str, tot_time):

         """Report download finished."""
         if self.params.get('noprogress', False):
             self.to_screen(u'[download] Download completed')
         else:
         """Report download finished."""
         if self.params.get('noprogress', False):
             self.to_screen(u'[download] Download completed')
         else:

-            self.to_screen(u'')
+            clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
+            self.to_screen(u'\r%s[download] 100%% of %s in %s' %
+                (clear_line, data_len_str, self.format_seconds(tot_time)))

 
     def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
     def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

+        test = self.params.get('test', False)

 
         # Check for rtmpdump first
         try:
 
         # Check for rtmpdump first
         try:


@@ -261,6 +292,8 @@ class FileDownloader(object):
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
             basic_args += ['--tcUrl', url]
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
             basic_args += ['--tcUrl', url]

+        if test:
+            basic_args += ['--stop', '1']

         args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
         args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:


@@ -270,7 +303,7 @@ class FileDownloader(object):
                 shell_quote = repr
             self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
         retval = subprocess.call(args)
                 shell_quote = repr
             self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
         retval = subprocess.call(args)

-        while retval == 2 or retval == 1:
+        while (retval == 2 or retval == 1) and not test:

             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
             time.sleep(5.0) # This seems to be needed
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
             time.sleep(5.0) # This seems to be needed


@@ -283,7 +316,7 @@ class FileDownloader(object):
                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = 0
                 break
                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = 0
                 break

-        if retval == 0:
+        if retval == 0 or (test and retval == 2):

             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
             self.try_rename(tmpfilename, filename)
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
             self.try_rename(tmpfilename, filename)


@@ -368,6 +401,7 @@ class FileDownloader(object):
             self._hook_progress({
                 'filename': filename,
                 'status': 'finished',
             self._hook_progress({
                 'filename': filename,
                 'status': 'finished',

+                'total_bytes': os.path.getsize(encodeFilename(filename)),

             })
             return True
 
             })
             return True
 


@@ -514,13 +548,14 @@ class FileDownloader(object):
                 block_size = self.best_block_size(after - before, len(data_block))
 
             # Progress message
                 block_size = self.best_block_size(after - before, len(data_block))
 
             # Progress message

-            speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)

             if data_len is None:
                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
             if data_len is None:
                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')

+                eta = None

             else:
             else:

-                percent_str = self.calc_percent(byte_counter, data_len)
-                eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
-                self.report_progress(percent_str, data_len_str, speed_str, eta_str)
+                percent = self.calc_percent(byte_counter, data_len)
+                eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+                self.report_progress(percent, data_len_str, speed, eta)

 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,
 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,


@@ -528,6 +563,8 @@ class FileDownloader(object):
                 'tmpfilename': tmpfilename,
                 'filename': filename,
                 'status': 'downloading',
                 'tmpfilename': tmpfilename,
                 'filename': filename,
                 'status': 'downloading',

+                'eta': eta,
+                'speed': speed,

             })
 
             # Apply rate limit
             })
 
             # Apply rate limit


@@ -538,7 +575,7 @@ class FileDownloader(object):
             self.report_error(u'Did not get any data blocks')
             return False
         stream.close()
             self.report_error(u'Did not get any data blocks')
             return False
         stream.close()

-        self.report_finish()
+        self.report_finish(data_len_str, (time.time() - start))

         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)
         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)


@@ -570,6 +607,8 @@ class FileDownloader(object):
         * downloaded_bytes: Bytes on disks
         * total_bytes: Total bytes, None if unknown
         * tmpfilename: The filename we're currently writing to
         * downloaded_bytes: Bytes on disks
         * total_bytes: Total bytes, None if unknown
         * tmpfilename: The filename we're currently writing to

+        * eta: The estimated time in seconds, None if unknown
+        * speed: The download speed in bytes/second, None if unknown

 
         Hooks are guaranteed to be called at least once (with status "finished")
         if the download is successful.
 
         Hooks are guaranteed to be called at least once (with status "finished")
         if the download is successful.