_ Git - youtube-dl/blob - youtube_dl/downloader/external.py

   1 from __future__ import unicode_literals
   2
   3 import os.path
   4 import subprocess
   5 import sys
   6
   7 from .common import FileDownloader
   8 from ..utils import (
   9     encodeFilename,
  10     std_headers,
  11 )
  12
  13
  14 class ExternalFD(FileDownloader):
  15     def real_download(self, filename, info_dict):
  16         self.report_destination(filename)
  17         tmpfilename = self.temp_name(filename)
  18
  19         retval = self._call_downloader(tmpfilename, info_dict)
  20         if retval == 0:
  21             fsize = os.path.getsize(encodeFilename(tmpfilename))
  22             self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
  23             self.try_rename(tmpfilename, filename)
  24             self._hook_progress({
  25                 'downloaded_bytes': fsize,
  26                 'total_bytes': fsize,
  27                 'filename': filename,
  28                 'status': 'finished',
  29             })
  30             return True
  31         else:
  32             self.to_stderr('\n')
  33             self.report_error('%s exited with code %d' % (
  34                 self.get_basename(), retval))
  35             return False
  36
  37     @classmethod
  38     def get_basename(cls):
  39         return cls.__name__[:-2].lower()
  40
  41     @property
  42     def exe(self):
  43         return self.params.get('external_downloader')
  44
  45     @classmethod
  46     def supports(cls, info_dict):
  47         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
  48
  49     def _calc_headers(self, info_dict):
  50         res = std_headers.copy()
  51
  52         ua = info_dict.get('user_agent')
  53         if ua is not None:
  54             res['User-Agent'] = ua
  55
  56         cookies = self._calc_cookies(info_dict)
  57         if cookies:
  58             res['Cookie'] = cookies
  59
  60         return res
  61
  62     def _calc_cookies(self, info_dict):
  63         class _PseudoRequest(object):
  64             def __init__(self, url):
  65                 self.url = url
  66                 self.headers = {}
  67                 self.unverifiable = False
  68
  69             def add_unredirected_header(self, k, v):
  70                 self.headers[k] = v
  71
  72             def get_full_url(self):
  73                 return self.url
  74
  75             def is_unverifiable(self):
  76                 return self.unverifiable
  77
  78             def has_header(self, h):
  79                 return h in self.headers
  80
  81         pr = _PseudoRequest(info_dict['url'])
  82         self.ydl.cookiejar.add_cookie_header(pr)
  83         return pr.headers.get('Cookie')
  84
  85     def _call_downloader(self, tmpfilename, info_dict):
  86         """ Either overwrite this or implement _make_cmd """
  87         cmd = self._make_cmd(tmpfilename, info_dict)
  88
  89         if sys.platform == 'win32' and sys.version_info < (3, 0):
  90             # Windows subprocess module does not actually support Unicode
  91             # on Python 2.x
  92             # See http://stackoverflow.com/a/9951851/35070
  93             subprocess_encoding = sys.getfilesystemencoding()
  94             cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
  95         else:
  96             subprocess_encoding = None
  97         self._debug_cmd(cmd, subprocess_encoding)
  98
  99         p = subprocess.Popen(
 100             cmd, stderr=subprocess.PIPE)
 101         _, stderr = p.communicate()
 102         if p.returncode != 0:
 103             self.to_stderr(stderr)
 104         return p.returncode
 105
 106
 107 class CurlFD(ExternalFD):
 108     def _make_cmd(self, tmpfilename, info_dict):
 109         cmd = [self.exe, '-o', tmpfilename]
 110         for key, val in self._calc_headers(info_dict).items():
 111             cmd += ['--header', '%s: %s' % (key, val)]
 112         cmd += ['--', info_dict['url']]
 113         return cmd
 114
 115
 116 class WgetFD(ExternalFD):
 117     def _make_cmd(self, tmpfilename, info_dict):
 118         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
 119         for key, val in self._calc_headers(info_dict).items():
 120             cmd += ['--header', '%s: %s' % (key, val)]
 121         cmd += ['--', info_dict['url']]
 122         return cmd
 123
 124
 125 class Aria2cFD(ExternalFD):
 126     def _make_cmd(self, tmpfilename, info_dict):
 127         cmd = [
 128             self.exe, '-c',
 129             '--min-split-size', '1M', '--max-connection-per-server', '4']
 130         dn = os.path.dirname(tmpfilename)
 131         if dn:
 132             cmd += ['--dir', dn]
 133         cmd += ['--out', os.path.basename(tmpfilename)]
 134         for key, val in self._calc_headers(info_dict).items():
 135             cmd += ['--header', '%s: %s' % (key, val)]
 136         cmd += ['--', info_dict['url']]
 137         return cmd
 138
 139 _BY_NAME = dict(
 140     (klass.get_basename(), klass)
 141     for name, klass in globals().items()
 142     if name.endswith('FD') and name != 'ExternalFD'
 143 )
 144
 145
 146 def list_external_downloaders():
 147     return sorted(_BY_NAME.keys())
 148
 149
 150 def get_external_downloader(external_downloader):
 151     """ Given the name of the executable, see whether we support the given
 152         downloader . """
 153     bn = os.path.basename(external_downloader)
 154     return _BY_NAME[bn]