Add new option --source-address

Closes #3618, fixes #721, fixes #2481, fixes #4551, closes #1020.

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cc3ec2fb408862ba454f8d47160ee54be2c2cf0..fd2c0e04406cd5babc5e620e1bdb2204fb55e04b 100755 (executable)
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -211,6 +211,7 @@ class YoutubeDL(object):
                        - "warn": only emit a warning
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise
                        - "warn": only emit a warning
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise

+    source_address:    (Experimental) Client-side IP address to bind to.

 
 
     The following parameters are not used by YoutubeDL itself, they are used by
 
 
     The following parameters are not used by YoutubeDL itself, they are used by


@@ -1493,9 +1494,8 @@ class YoutubeDL(object):
         proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0

-        https_handler = make_HTTPS_handler(
-            self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
-        ydlh = YoutubeDLHandler(debuglevel=debuglevel)
+        https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
+        ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)

         opener = compat_urllib_request.build_opener(
             https_handler, proxy_handler, cookie_processor, ydlh)
         # Delete the default user-agent header, which would otherwise apply in
         opener = compat_urllib_request.build_opener(
             https_handler, proxy_handler, cookie_processor, ydlh)
         # Delete the default user-agent header, which would otherwise apply in

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 659a92a3b944c25f40e2ad86aa43f9db37fbfe9a..d74a304b7b42063e9332c42acfaea92c3af472b0 100644 (file)
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -327,6 +327,7 @@ def _real_main(argv=None):
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
         'fixup': opts.fixup,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
         'fixup': opts.fixup,

+        'source_address': opts.source_address,

     }
 
     with YoutubeDL(ydl_opts) as ydl:
     }
 
     with YoutubeDL(ydl_opts) as ydl:

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 46d438846787e9a95ea8631c869f96341d4ccb9c..44a902573e56e2df184ff6d677d49dc265f7973e 100644 (file)
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -4,6 +4,7 @@ import getpass
 import optparse
 import os
 import re
 import optparse
 import os
 import re

+import socket

 import subprocess
 import sys
 
 import subprocess
 import sys
 


@@ -307,6 +308,32 @@ else:
     compat_kwargs = lambda kwargs: kwargs
 
 
     compat_kwargs = lambda kwargs: kwargs
 
 

+if sys.version_info < (2, 7):
+    def compat_socket_create_connection(address, timeout, source_address=None):
+        host, port = address
+        err = None
+        for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
+            af, socktype, proto, canonname, sa = res
+            sock = None
+            try:
+                sock = socket.socket(af, socktype, proto)
+                sock.settimeout(timeout)
+                if source_address:
+                    sock.bind(source_address)
+                sock.connect(sa)
+                return sock
+            except socket.error as _:
+                err = _
+                if sock is not None:
+                    sock.close()
+        if err is not None:
+            raise err
+        else:
+            raise error("getaddrinfo returns an empty list")
+else:
+    compat_socket_create_connection = socket.create_connection
+
+

 # Fix https://github.com/rg3/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
 # Fix https://github.com/rg3/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():


@@ -343,6 +370,7 @@ __all__ = [
     'compat_parse_qs',
     'compat_print',
     'compat_str',
     'compat_parse_qs',
     'compat_print',
     'compat_str',

+    'compat_socket_create_connection',

     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e5602bb3a3787fae92da14731e7f3c9922066fcd..31351d43dfc23ff72cb340efc964616b68a3b1a6 100644 (file)
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -148,14 +148,6 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')

-    general.add_option(
-        '--proxy', dest='proxy',
-        default=None, metavar='URL',
-        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
-    general.add_option(
-        '--socket-timeout',
-        dest='socket_timeout', type=float, default=None,
-        help='Time to wait before giving up, in seconds')

     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',


@@ -173,6 +165,21 @@ def parseOpts(overrideArguments=None):
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
 
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
 

+    network = optparse.OptionGroup(parser, 'Network Options')
+    network.add_option(
+        '--proxy', dest='proxy',
+        default=None, metavar='URL',
+        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
+    network.add_option(
+        '--socket-timeout',
+        dest='socket_timeout', type=float, default=None, metavar='SECONDS',
+        help='Time to wait before giving up, in seconds')
+    network.add_option(
+        '--source-address',
+        metavar='IP', dest='source_address', default=None,
+        help='Client-side IP address to bind to (experimental)',
+    )
+

     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
         '--playlist-start',
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
         '--playlist-start',


@@ -652,6 +659,7 @@ def parseOpts(overrideArguments=None):
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
 
     parser.add_option_group(general)
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
 
     parser.add_option_group(general)

+    parser.add_option_group(network)

     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 3f9c5249dbc1b3e2d078db5254b4899b167965eb..d8be4049f5dce0fdd9a61f2aff3c4284d494e598 100644 (file)
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -59,7 +59,7 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 

-    https_handler = make_HTTPS_handler(False)
+    https_handler = make_HTTPS_handler({})

     opener = compat_urllib_request.build_opener(https_handler)
 
     # Check if there is a new version
     opener = compat_urllib_request.build_opener(https_handler)
 
     # Check if there is a new version

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a12b0a7de4b00e7879ea4af75d2b4f333dc8f53a..cc5f510f40fff66ce374f2586aae43686a240b7d 100644 (file)
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -10,6 +10,7 @@ import ctypes
 import datetime
 import email.utils
 import errno
 import datetime
 import email.utils
 import errno

+import functools

 import gzip
 import itertools
 import io
 import gzip
 import itertools
 import io


@@ -34,7 +35,9 @@ from .compat import (
     compat_chr,
     compat_getenv,
     compat_html_entities,
     compat_chr,
     compat_getenv,
     compat_html_entities,

+    compat_http_client,

     compat_parse_qs,
     compat_parse_qs,

+    compat_socket_create_connection,

     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,


@@ -391,13 +394,14 @@ def formatSeconds(secs):
         return '%d' % secs
 
 
         return '%d' % secs
 
 

-def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
+def make_HTTPS_handler(params, **kwargs):
+    opts_no_check_certificate = params.get('nocheckcertificate', False)

     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
         try:
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
         try:

-            return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+            return YoutubeDLHTTPSHandler(params, context=context, **kwargs)

         except TypeError:
             # Python 2.7.8
             # (create_default_context present but HTTPSHandler has no context=)
         except TypeError:
             # Python 2.7.8
             # (create_default_context present but HTTPSHandler has no context=)


@@ -420,17 +424,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
                 except ssl.SSLError:
                     self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
 
                 except ssl.SSLError:
                     self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
 

-        class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler):
-            def https_open(self, req):
-                return self.do_open(HTTPSConnectionV3, req)
-        return HTTPSHandlerV3(**kwargs)
+        return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs)

     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()
     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()

-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+        return YoutubeDLHTTPSHandler(params, context=context, **kwargs)

 
 
 class ExtractorError(Exception):
 
 
 class ExtractorError(Exception):


@@ -544,6 +545,26 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
         self.expected = expected
 
 

+def _create_http_connection(ydl_handler, http_class, is_https=False, *args, **kwargs):
+    hc = http_class(*args, **kwargs)
+    source_address = ydl_handler._params.get('source_address')
+    if source_address is not None:
+        sa = (source_address, 0)
+        if hasattr(hc, 'source_address'):  # Python 2.7+
+            hc.source_address = sa
+        else:  # Python 2.6
+            def _hc_connect(self, *args, **kwargs):
+                sock = compat_socket_create_connection(
+                    (self.host, self.port), self.timeout, sa)
+                if is_https:
+                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                else:
+                    self.sock = sock
+            hc.connect = functools.partial(_hc_connect, hc)
+
+    return hc
+
+

 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 


@@ -562,6 +583,15 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     public domain.
     """
 
     public domain.
     """
 

+    def __init__(self, params, *args, **kwargs):
+        compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
+        self._params = params
+
+    def http_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, compat_http_client.HTTPConnection),
+            req)
+

     @staticmethod
     def deflate(data):
         try:
     @staticmethod
     def deflate(data):
         try:


@@ -631,6 +661,18 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_response = http_response
 
 
     https_response = http_response
 
 

+class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
+    def __init__(self, params, https_conn_class=None, *args, **kwargs):
+        compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
+        self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
+        self._params = params
+
+    def https_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, self._https_conn_class, True),
+            req)
+
+

 def parse_iso8601(date_str, delimiter='T'):
     """ Return a UNIX timestamp from the given date """
 
 def parse_iso8601(date_str, delimiter='T'):
     """ Return a UNIX timestamp from the given date """