[downloader/external] Respect --no-check-certificate for curl and aria2c and --proxy...
[youtube-dl] / youtube_dl / downloader / external.py
1 from __future__ import unicode_literals
2
3 import os.path
4 import subprocess
5
6 from .common import FileDownloader
7 from ..utils import (
8     encodeFilename,
9     encodeArgument,
10 )
11
12
13 class ExternalFD(FileDownloader):
14     def real_download(self, filename, info_dict):
15         self.report_destination(filename)
16         tmpfilename = self.temp_name(filename)
17
18         retval = self._call_downloader(tmpfilename, info_dict)
19         if retval == 0:
20             fsize = os.path.getsize(encodeFilename(tmpfilename))
21             self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
22             self.try_rename(tmpfilename, filename)
23             self._hook_progress({
24                 'downloaded_bytes': fsize,
25                 'total_bytes': fsize,
26                 'filename': filename,
27                 'status': 'finished',
28             })
29             return True
30         else:
31             self.to_stderr('\n')
32             self.report_error('%s exited with code %d' % (
33                 self.get_basename(), retval))
34             return False
35
36     @classmethod
37     def get_basename(cls):
38         return cls.__name__[:-2].lower()
39
40     @property
41     def exe(self):
42         return self.params.get('external_downloader')
43
44     @classmethod
45     def supports(cls, info_dict):
46         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
47
48     def _option(self, command_option, param):
49         param = self.params.get(param)
50         if param is None or param is False:
51             return []
52         if isinstance(param, bool):
53             return [command_option]
54         return [command_option, param]
55
56     def _configuration_args(self, default=[]):
57         ex_args = self.params.get('external_downloader_args')
58         if ex_args is None:
59             return default
60         assert isinstance(ex_args, list)
61         return ex_args
62
63     def _call_downloader(self, tmpfilename, info_dict):
64         """ Either overwrite this or implement _make_cmd """
65         cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
66
67         self._debug_cmd(cmd)
68
69         p = subprocess.Popen(
70             cmd, stderr=subprocess.PIPE)
71         _, stderr = p.communicate()
72         if p.returncode != 0:
73             self.to_stderr(stderr)
74         return p.returncode
75
76
77 class CurlFD(ExternalFD):
78     def _make_cmd(self, tmpfilename, info_dict):
79         cmd = [self.exe, '--location', '-o', tmpfilename]
80         for key, val in info_dict['http_headers'].items():
81             cmd += ['--header', '%s: %s' % (key, val)]
82         cmd += self._option('--interface', 'source_address')
83         cmd += self._option('--proxy', 'proxy')
84         cmd += self._option('--insecure', 'nocheckcertificate')
85         cmd += self._configuration_args()
86         cmd += ['--', info_dict['url']]
87         return cmd
88
89
90 class AxelFD(ExternalFD):
91     def _make_cmd(self, tmpfilename, info_dict):
92         cmd = [self.exe, '-o', tmpfilename]
93         for key, val in info_dict['http_headers'].items():
94             cmd += ['-H', '%s: %s' % (key, val)]
95         cmd += self._configuration_args()
96         cmd += ['--', info_dict['url']]
97         return cmd
98
99
100 class WgetFD(ExternalFD):
101     def _make_cmd(self, tmpfilename, info_dict):
102         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
103         for key, val in info_dict['http_headers'].items():
104             cmd += ['--header', '%s: %s' % (key, val)]
105         cmd += self._option('--bind-address', 'source_address')
106         cmd += self._option('--proxy', 'proxy')
107         cmd += self._option('--no-check-certificate', 'nocheckcertificate')
108         cmd += self._configuration_args()
109         cmd += ['--', info_dict['url']]
110         return cmd
111
112
113 class Aria2cFD(ExternalFD):
114     def _make_cmd(self, tmpfilename, info_dict):
115         cmd = [self.exe, '-c']
116         cmd += self._configuration_args([
117             '--min-split-size', '1M', '--max-connection-per-server', '4'])
118         dn = os.path.dirname(tmpfilename)
119         if dn:
120             cmd += ['--dir', dn]
121         cmd += ['--out', os.path.basename(tmpfilename)]
122         for key, val in info_dict['http_headers'].items():
123             cmd += ['--header', '%s: %s' % (key, val)]
124         cmd += self._option('--interface', 'source_address')
125         cmd += self._option('--all-proxy', 'proxy')
126         cmd += self._option('--check-certificate=false', 'nocheckcertificate')
127         cmd += ['--', info_dict['url']]
128         return cmd
129
130
131 class HttpieFD(ExternalFD):
132     def _make_cmd(self, tmpfilename, info_dict):
133         cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
134         for key, val in info_dict['http_headers'].items():
135             cmd += ['%s:%s' % (key, val)]
136         return cmd
137
138 _BY_NAME = dict(
139     (klass.get_basename(), klass)
140     for name, klass in globals().items()
141     if name.endswith('FD') and name != 'ExternalFD'
142 )
143
144
145 def list_external_downloaders():
146     return sorted(_BY_NAME.keys())
147
148
149 def get_external_downloader(external_downloader):
150     """ Given the name of the executable, see whether we support the given
151         downloader . """
152     # Drop .exe extension on Windows
153     bn = os.path.splitext(os.path.basename(external_downloader))[0]
154     return _BY_NAME[bn]