Fix typos

[youtube-dl] / youtube_dl / extractor / common.py

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 71bdcad5ae34e47690ec70ac7873e6d9960b894b..b05b22a94b0bac346e776ef5c3d42305cf5f4163 100644 (file)
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -18,8 +18,6 @@ from ..compat import (
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse,

-    compat_urllib_parse_urlparse,
-    compat_urllib_request,

     compat_urlparse,
     compat_str,
     compat_etree_fromstring,
     compat_urlparse,
     compat_str,
     compat_etree_fromstring,


@@ -31,17 +29,20 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     determine_ext,
     clean_html,
     compiled_regex_type,
     determine_ext,

+    error_to_compat_str,

     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,

+    sanitized_Request,

     unescapeHTML,
     unified_strdate,
     url_basename,
     xpath_text,
     xpath_with_ns,
     unescapeHTML,
     unified_strdate,
     url_basename,
     xpath_text,
     xpath_with_ns,

+    determine_protocol,

 )
 
 
 )
 
 


@@ -107,8 +108,9 @@ class InfoExtractor(object):
                                  -2 or smaller for less than default.
                                  < -1000 to hide the format (if there is
                                     another one which is strictly better)
                                  -2 or smaller for less than default.
                                  < -1000 to hide the format (if there is
                                     another one which is strictly better)

-                    * language_preference  Is this in the correct requested
-                                 language?
+                    * language   Language code, e.g. "de" or "en-US".
+                    * language_preference  Is this in the language mentioned in
+                                 the URL?

                                  10 if it's what the URL is about,
                                  -1 for default (don't know),
                                  -10 otherwise, other values reserved for now.
                                  10 if it's what the URL is about,
                                  -1 for default (don't know),
                                  -10 otherwise, other values reserved for now.


@@ -167,7 +169,7 @@ class InfoExtractor(object):
                     "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
                     "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions

-    duration:       Length of the video in seconds, as an integer.
+    duration:       Length of the video in seconds, as an integer or float.

     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video


@@ -199,6 +201,26 @@ class InfoExtractor(object):
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
 
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
 

+    The following fields should only be used when the video belongs to some logical
+    chapter or section:
+
+    chapter:        Name or title of the chapter the video belongs to.
+    chapter_number: Number of the chapter the video belongs to, as an integer.
+    chapter_id:     Id of the chapter the video belongs to, as a unicode string.
+
+    The following fields should only be used when the video is an episode of some
+    series or programme:
+
+    series:         Title of the series or programme the video episode belongs to.
+    season:         Title of the season the video episode belongs to.
+    season_number:  Number of the season the video episode belongs to, as an integer.
+    season_id:      Id of the season the video episode belongs to, as a unicode string.
+    episode:        Title of the video episode. Unlike mandatory video title field,
+                    this field should denote the exact title of the video episode
+                    without any kind of decoration.
+    episode_number: Number of the video episode within a season, as an integer.
+    episode_id:     Id of the video episode, as a unicode string.
+

     Unless mentioned otherwise, the fields should be Unicode strings.
 
     Unless mentioned otherwise, None is equivalent to absence of information.
     Unless mentioned otherwise, the fields should be Unicode strings.
 
     Unless mentioned otherwise, None is equivalent to absence of information.


@@ -291,9 +313,9 @@ class InfoExtractor(object):
         except ExtractorError:
             raise
         except compat_http_client.IncompleteRead as e:
         except ExtractorError:
             raise
         except compat_http_client.IncompleteRead as e:

-            raise ExtractorError('A network error has occured.', cause=e, expected=True)
+            raise ExtractorError('A network error has occurred.', cause=e, expected=True)

         except (KeyError, StopIteration) as e:
         except (KeyError, StopIteration) as e:

-            raise ExtractorError('An extractor error has occured.', cause=e)
+            raise ExtractorError('An extractor error has occurred.', cause=e)

 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""
 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""


@@ -332,7 +354,8 @@ class InfoExtractor(object):
                 return False
             if errnote is None:
                 errnote = 'Unable to download webpage'
                 return False
             if errnote is None:
                 errnote = 'Unable to download webpage'

-            errmsg = '%s: %s' % (errnote, compat_str(err))
+
+            errmsg = '%s: %s' % (errnote, error_to_compat_str(err))

             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:


@@ -622,7 +645,7 @@ class InfoExtractor(object):
                 else:
                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
             except (IOError, netrc.NetrcParseError) as err:
                 else:
                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
             except (IOError, netrc.NetrcParseError) as err:

-                self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
+                self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))

 
         return (username, password)
 
 
         return (username, password)
 


@@ -776,14 +799,12 @@ class InfoExtractor(object):
 
             preference = f.get('preference')
             if preference is None:
 
             preference = f.get('preference')
             if preference is None:

-                proto = f.get('protocol')
-                if proto is None:
-                    proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
-
-                preference = 0 if proto in ['http', 'https'] else -0.1
+                preference = 0

                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
                     preference -= 0.5
 
                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
                     preference -= 0.5
 

+            proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
+

             if f.get('vcodec') == 'none':  # audio only
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
             if f.get('vcodec') == 'none':  # audio only
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']


@@ -814,6 +835,7 @@ class InfoExtractor(object):
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,

+                proto_preference,

                 ext_preference,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 ext_preference,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,


@@ -883,7 +905,7 @@ class InfoExtractor(object):
             fatal=fatal)
 
         if manifest is False:
             fatal=fatal)
 
         if manifest is False:

-            return manifest
+            return []

 
         formats = []
         manifest_version = '1.0'
 
         formats = []
         manifest_version = '1.0'


@@ -909,10 +931,8 @@ class InfoExtractor(object):
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':

-                    f4m_formats = self._extract_f4m_formats(
-                        manifest_url, video_id, preference, f4m_id, fatal=fatal)
-                    if f4m_formats:
-                        formats.extend(f4m_formats)
+                    formats.extend(self._extract_f4m_formats(
+                        manifest_url, video_id, preference, f4m_id, fatal=fatal))

                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({


@@ -954,7 +974,7 @@ class InfoExtractor(object):
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
         if res is False:
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
         if res is False:

-            return res
+            return []

         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
         last_info = None
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
         last_info = None


@@ -1146,10 +1166,8 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':

-                m3u8_formats = self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False))

                 continue
 
             if src_ext == 'f4m':
                 continue
 
             if src_ext == 'f4m':


@@ -1161,9 +1179,7 @@ class InfoExtractor(object):
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
                 f4m_url += compat_urllib_parse.urlencode(f4m_params)
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
                 f4m_url += compat_urllib_parse.urlencode(f4m_params)

-                f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))

                 continue
 
             if src_url.startswith('http') and self._is_valid_url(src, video_id):
                 continue
 
             if src_url.startswith('http') and self._is_valid_url(src, video_id):


@@ -1285,7 +1301,7 @@ class InfoExtractor(object):
 
     def _get_cookies(self, url):
         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
 
     def _get_cookies(self, url):
         """ Return a compat_cookies.SimpleCookie with the cookies for the url """

-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)

         self._downloader.cookiejar.add_cookie_header(req)
         return compat_cookies.SimpleCookie(req.get_header('Cookie'))
 
         self._downloader.cookiejar.add_cookie_header(req)
         return compat_cookies.SimpleCookie(req.get_header('Cookie'))