_ Git - youtube-dl/blob - youtube_dl/extractor/common.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import base64
   5 import datetime
   6 import hashlib
   7 import json
   8 import netrc
   9 import os
  10 import random
  11 import re
  12 import socket
  13 import ssl
  14 import sys
  15 import time
  16 import math
  17
  18 from ..compat import (
  19     compat_cookiejar_Cookie,
  20     compat_cookies,
  21     compat_etree_Element,
  22     compat_etree_fromstring,
  23     compat_getpass,
  24     compat_integer_types,
  25     compat_http_client,
  26     compat_os_name,
  27     compat_str,
  28     compat_urllib_error,
  29     compat_urllib_parse_unquote,
  30     compat_urllib_parse_urlencode,
  31     compat_urllib_request,
  32     compat_urlparse,
  33     compat_xml_parse_error,
  34 )
  35 from ..downloader.f4m import (
  36     get_base_url,
  37     remove_encrypted_media,
  38 )
  39 from ..utils import (
  40     NO_DEFAULT,
  41     age_restricted,
  42     base_url,
  43     bug_reports_message,
  44     clean_html,
  45     compiled_regex_type,
  46     determine_ext,
  47     determine_protocol,
  48     dict_get,
  49     error_to_compat_str,
  50     ExtractorError,
  51     extract_attributes,
  52     fix_xml_ampersands,
  53     float_or_none,
  54     GeoRestrictedError,
  55     GeoUtils,
  56     int_or_none,
  57     js_to_json,
  58     JSON_LD_RE,
  59     mimetype2ext,
  60     orderedSet,
  61     parse_bitrate,
  62     parse_codecs,
  63     parse_duration,
  64     parse_iso8601,
  65     parse_m3u8_attributes,
  66     parse_resolution,
  67     RegexNotFoundError,
  68     sanitized_Request,
  69     sanitize_filename,
  70     str_or_none,
  71     str_to_int,
  72     strip_or_none,
  73     unescapeHTML,
  74     unified_strdate,
  75     unified_timestamp,
  76     update_Request,
  77     update_url_query,
  78     urljoin,
  79     url_basename,
  80     url_or_none,
  81     xpath_element,
  82     xpath_text,
  83     xpath_with_ns,
  84 )
  85
  86
  87 class InfoExtractor(object):
  88     """Information Extractor class.
  89
  90     Information extractors are the classes that, given a URL, extract
  91     information about the video (or videos) the URL refers to. This
  92     information includes the real video URL, the video title, author and
  93     others. The information is stored in a dictionary which is then
  94     passed to the YoutubeDL. The YoutubeDL processes this
  95     information possibly downloading the video to the file system, among
  96     other possible outcomes.
  97
  98     The type field determines the type of the result.
  99     By far the most common value (and the default if _type is missing) is
 100     "video", which indicates a single video.
 101
 102     For a video, the dictionaries must include the following fields:
 103
 104     id:             Video identifier.
 105     title:          Video title, unescaped.
 106
 107     Additionally, it must contain either a formats entry or a url one:
 108
 109     formats:        A list of dictionaries for each format available, ordered
 110                     from worst to best quality.
 111
 112                     Potential fields:
 113                     * url        The mandatory URL representing the media:
 114                                    for plain file media - HTTP URL of this file,
 115                                    for RTMP - RTMP URL,
 116                                    for HLS - URL of the M3U8 media playlist,
 117                                    for HDS - URL of the F4M manifest,
 118                                    for DASH
 119                                      - HTTP URL to plain file media (in case of
 120                                        unfragmented media)
 121                                      - URL of the MPD manifest or base URL
 122                                        representing the media if MPD manifest
 123                                        is parsed from a string (in case of
 124                                        fragmented media)
 125                                    for MSS - URL of the ISM manifest.
 126                     * manifest_url
 127                                  The URL of the manifest file in case of
 128                                  fragmented media:
 129                                    for HLS - URL of the M3U8 master playlist,
 130                                    for HDS - URL of the F4M manifest,
 131                                    for DASH - URL of the MPD manifest,
 132                                    for MSS - URL of the ISM manifest.
 133                     * ext        Will be calculated from URL if missing
 134                     * format     A human-readable description of the format
 135                                  ("mp4 container with h264/opus").
 136                                  Calculated from the format_id, width, height.
 137                                  and format_note fields if missing.
 138                     * format_id  A short description of the format
 139                                  ("mp4_h264_opus" or "19").
 140                                 Technically optional, but strongly recommended.
 141                     * format_note Additional info about the format
 142                                  ("3D" or "DASH video")
 143                     * width      Width of the video, if known
 144                     * height     Height of the video, if known
 145                     * resolution Textual description of width and height
 146                     * tbr        Average bitrate of audio and video in KBit/s
 147                     * abr        Average audio bitrate in KBit/s
 148                     * acodec     Name of the audio codec in use
 149                     * asr        Audio sampling rate in Hertz
 150                     * vbr        Average video bitrate in KBit/s
 151                     * fps        Frame rate
 152                     * vcodec     Name of the video codec in use
 153                     * container  Name of the container format
 154                     * filesize   The number of bytes, if known in advance
 155                     * filesize_approx  An estimate for the number of bytes
 156                     * player_url SWF Player URL (used for rtmpdump).
 157                     * protocol   The protocol that will be used for the actual
 158                                  download, lower-case.
 159                                  "http", "https", "rtsp", "rtmp", "rtmpe",
 160                                  "m3u8", "m3u8_native" or "http_dash_segments".
 161                     * fragment_base_url
 162                                  Base URL for fragments. Each fragment's path
 163                                  value (if present) will be relative to
 164                                  this URL.
 165                     * fragments  A list of fragments of a fragmented media.
 166                                  Each fragment entry must contain either an url
 167                                  or a path. If an url is present it should be
 168                                  considered by a client. Otherwise both path and
 169                                  fragment_base_url must be present. Here is
 170                                  the list of all potential fields:
 171                                  * "url" - fragment's URL
 172                                  * "path" - fragment's path relative to
 173                                             fragment_base_url
 174                                  * "duration" (optional, int or float)
 175                                  * "filesize" (optional, int)
 176                     * preference Order number of this format. If this field is
 177                                  present and not None, the formats get sorted
 178                                  by this field, regardless of all other values.
 179                                  -1 for default (order by other properties),
 180                                  -2 or smaller for less than default.
 181                                  < -1000 to hide the format (if there is
 182                                     another one which is strictly better)
 183                     * language   Language code, e.g. "de" or "en-US".
 184                     * language_preference  Is this in the language mentioned in
 185                                  the URL?
 186                                  10 if it's what the URL is about,
 187                                  -1 for default (don't know),
 188                                  -10 otherwise, other values reserved for now.
 189                     * quality    Order number of the video quality of this
 190                                  format, irrespective of the file format.
 191                                  -1 for default (order by other properties),
 192                                  -2 or smaller for less than default.
 193                     * source_preference  Order number for this video source
 194                                   (quality takes higher priority)
 195                                  -1 for default (order by other properties),
 196                                  -2 or smaller for less than default.
 197                     * http_headers  A dictionary of additional HTTP headers
 198                                  to add to the request.
 199                     * stretched_ratio  If given and not 1, indicates that the
 200                                  video's pixels are not square.
 201                                  width : height ratio as float.
 202                     * no_resume  The server does not support resuming the
 203                                  (HTTP or RTMP) download. Boolean.
 204                     * downloader_options  A dictionary of downloader options as
 205                                  described in FileDownloader
 206
 207     url:            Final video URL.
 208     ext:            Video filename extension.
 209     format:         The video format, defaults to ext (used for --get-format)
 210     player_url:     SWF Player URL (used for rtmpdump).
 211
 212     The following fields are optional:
 213
 214     alt_title:      A secondary title of the video.
 215     display_id      An alternative identifier for the video, not necessarily
 216                     unique, but available before title. Typically, id is
 217                     something like "4234987", title "Dancing naked mole rats",
 218                     and display_id "dancing-naked-mole-rats"
 219     thumbnails:     A list of dictionaries, with the following entries:
 220                         * "id" (optional, string) - Thumbnail format ID
 221                         * "url"
 222                         * "preference" (optional, int) - quality of the image
 223                         * "width" (optional, int)
 224                         * "height" (optional, int)
 225                         * "resolution" (optional, string "{width}x{height}",
 226                                         deprecated)
 227                         * "filesize" (optional, int)
 228     thumbnail:      Full URL to a video thumbnail image.
 229     description:    Full video description.
 230     uploader:       Full name of the video uploader.
 231     license:        License name the video is licensed under.
 232     creator:        The creator of the video.
 233     release_date:   The date (YYYYMMDD) when the video was released.
 234     timestamp:      UNIX timestamp of the moment the video became available.
 235     upload_date:    Video upload date (YYYYMMDD).
 236                     If not explicitly set, calculated from timestamp.
 237     uploader_id:    Nickname or id of the video uploader.
 238     uploader_url:   Full URL to a personal webpage of the video uploader.
 239     channel:        Full name of the channel the video is uploaded on.
 240                     Note that channel fields may or may not repeat uploader
 241                     fields. This depends on a particular extractor.
 242     channel_id:     Id of the channel.
 243     channel_url:    Full URL to a channel webpage.
 244     location:       Physical location where the video was filmed.
 245     subtitles:      The available subtitles as a dictionary in the format
 246                     {tag: subformats}. "tag" is usually a language code, and
 247                     "subformats" is a list sorted from lower to higher
 248                     preference, each element is a dictionary with the "ext"
 249                     entry and one of:
 250                         * "data": The subtitles file contents
 251                         * "url": A URL pointing to the subtitles file
 252                     "ext" will be calculated from URL if missing
 253     automatic_captions: Like 'subtitles', used by the YoutubeIE for
 254                     automatically generated captions
 255     duration:       Length of the video in seconds, as an integer or float.
 256     view_count:     How many users have watched the video on the platform.
 257     like_count:     Number of positive ratings of the video
 258     dislike_count:  Number of negative ratings of the video
 259     repost_count:   Number of reposts of the video
 260     average_rating: Average rating give by users, the scale used depends on the webpage
 261     comment_count:  Number of comments on the video
 262     comments:       A list of comments, each with one or more of the following
 263                     properties (all but one of text or html optional):
 264                         * "author" - human-readable name of the comment author
 265                         * "author_id" - user ID of the comment author
 266                         * "id" - Comment ID
 267                         * "html" - Comment as HTML
 268                         * "text" - Plain text of the comment
 269                         * "timestamp" - UNIX timestamp of comment
 270                         * "parent" - ID of the comment this one is replying to.
 271                                      Set to "root" to indicate that this is a
 272                                      comment to the original video.
 273     age_limit:      Age restriction for the video, as an integer (years)
 274     webpage_url:    The URL to the video webpage, if given to youtube-dl it
 275                     should allow to get the same result again. (It will be set
 276                     by YoutubeDL if it's missing)
 277     categories:     A list of categories that the video falls in, for example
 278                     ["Sports", "Berlin"]
 279     tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
 280     is_live:        True, False, or None (=unknown). Whether this video is a
 281                     live stream that goes on instead of a fixed-length video.
 282     start_time:     Time in seconds where the reproduction should start, as
 283                     specified in the URL.
 284     end_time:       Time in seconds where the reproduction should end, as
 285                     specified in the URL.
 286     chapters:       A list of dictionaries, with the following entries:
 287                         * "start_time" - The start time of the chapter in seconds
 288                         * "end_time" - The end time of the chapter in seconds
 289                         * "title" (optional, string)
 290
 291     The following fields should only be used when the video belongs to some logical
 292     chapter or section:
 293
 294     chapter:        Name or title of the chapter the video belongs to.
 295     chapter_number: Number of the chapter the video belongs to, as an integer.
 296     chapter_id:     Id of the chapter the video belongs to, as a unicode string.
 297
 298     The following fields should only be used when the video is an episode of some
 299     series, programme or podcast:
 300
 301     series:         Title of the series or programme the video episode belongs to.
 302     season:         Title of the season the video episode belongs to.
 303     season_number:  Number of the season the video episode belongs to, as an integer.
 304     season_id:      Id of the season the video episode belongs to, as a unicode string.
 305     episode:        Title of the video episode. Unlike mandatory video title field,
 306                     this field should denote the exact title of the video episode
 307                     without any kind of decoration.
 308     episode_number: Number of the video episode within a season, as an integer.
 309     episode_id:     Id of the video episode, as a unicode string.
 310
 311     The following fields should only be used when the media is a track or a part of
 312     a music album:
 313
 314     track:          Title of the track.
 315     track_number:   Number of the track within an album or a disc, as an integer.
 316     track_id:       Id of the track (useful in case of custom indexing, e.g. 6.iii),
 317                     as a unicode string.
 318     artist:         Artist(s) of the track.
 319     genre:          Genre(s) of the track.
 320     album:          Title of the album the track belongs to.
 321     album_type:     Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc).
 322     album_artist:   List of all artists appeared on the album (e.g.
 323                     "Ash Borer / Fell Voices" or "Various Artists", useful for splits
 324                     and compilations).
 325     disc_number:    Number of the disc or other physical medium the track belongs to,
 326                     as an integer.
 327     release_year:   Year (YYYY) when the album was released.
 328
 329     Unless mentioned otherwise, the fields should be Unicode strings.
 330
 331     Unless mentioned otherwise, None is equivalent to absence of information.
 332
 333
 334     _type "playlist" indicates multiple videos.
 335     There must be a key "entries", which is a list, an iterable, or a PagedList
 336     object, each element of which is a valid dictionary by this specification.
 337
 338     Additionally, playlists can have "id", "title", "description", "uploader",
 339     "uploader_id", "uploader_url" attributes with the same semantics as videos
 340     (see above).
 341
 342
 343     _type "multi_video" indicates that there are multiple videos that
 344     form a single show, for examples multiple acts of an opera or TV episode.
 345     It must have an entries key like a playlist and contain all the keys
 346     required for a video at the same time.
 347
 348
 349     _type "url" indicates that the video must be extracted from another
 350     location, possibly by a different extractor. Its only required key is:
 351     "url" - the next URL to extract.
 352     The key "ie_key" can be set to the class name (minus the trailing "IE",
 353     e.g. "Youtube") if the extractor class is known in advance.
 354     Additionally, the dictionary may have any properties of the resolved entity
 355     known in advance, for example "title" if the title of the referred video is
 356     known ahead of time.
 357
 358
 359     _type "url_transparent" entities have the same specification as "url", but
 360     indicate that the given additional information is more precise than the one
 361     associated with the resolved URL.
 362     This is useful when a site employs a video service that hosts the video and
 363     its technical metadata, but that video service does not embed a useful
 364     title, description etc.
 365
 366
 367     Subclasses of this one should re-define the _real_initialize() and
 368     _real_extract() methods and define a _VALID_URL regexp.
 369     Probably, they should also be added to the list of extractors.
 370
 371     _GEO_BYPASS attribute may be set to False in order to disable
 372     geo restriction bypass mechanisms for a particular extractor.
 373     Though it won't disable explicit geo restriction bypass based on
 374     country code provided with geo_bypass_country.
 375
 376     _GEO_COUNTRIES attribute may contain a list of presumably geo unrestricted
 377     countries for this extractor. One of these countries will be used by
 378     geo restriction bypass mechanism right away in order to bypass
 379     geo restriction, of course, if the mechanism is not disabled.
 380
 381     _GEO_IP_BLOCKS attribute may contain a list of presumably geo unrestricted
 382     IP blocks in CIDR notation for this extractor. One of these IP blocks
 383     will be used by geo restriction bypass mechanism similarly
 384     to _GEO_COUNTRIES.
 385
 386     Finally, the _WORKING attribute should be set to False for broken IEs
 387     in order to warn the users and skip the tests.
 388     """
 389
 390     _ready = False
 391     _downloader = None
 392     _x_forwarded_for_ip = None
 393     _GEO_BYPASS = True
 394     _GEO_COUNTRIES = None
 395     _GEO_IP_BLOCKS = None
 396     _WORKING = True
 397
 398     def __init__(self, downloader=None):
 399         """Constructor. Receives an optional downloader."""
 400         self._ready = False
 401         self._x_forwarded_for_ip = None
 402         self.set_downloader(downloader)
 403
 404     @classmethod
 405     def suitable(cls, url):
 406         """Receives a URL and returns True if suitable for this IE."""
 407
 408         # This does not use has/getattr intentionally - we want to know whether
 409         # we have cached the regexp for *this* class, whereas getattr would also
 410         # match the superclass
 411         if '_VALID_URL_RE' not in cls.__dict__:
 412             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 413         return cls._VALID_URL_RE.match(url) is not None
 414
 415     @classmethod
 416     def _match_id(cls, url):
 417         if '_VALID_URL_RE' not in cls.__dict__:
 418             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 419         m = cls._VALID_URL_RE.match(url)
 420         assert m
 421         return compat_str(m.group('id'))
 422
 423     @classmethod
 424     def working(cls):
 425         """Getter method for _WORKING."""
 426         return cls._WORKING
 427
 428     def initialize(self):
 429         """Initializes an instance (authentication, etc)."""
 430         self._initialize_geo_bypass({
 431             'countries': self._GEO_COUNTRIES,
 432             'ip_blocks': self._GEO_IP_BLOCKS,
 433         })
 434         if not self._ready:
 435             self._real_initialize()
 436             self._ready = True
 437
 438     def _initialize_geo_bypass(self, geo_bypass_context):
 439         """
 440         Initialize geo restriction bypass mechanism.
 441
 442         This method is used to initialize geo bypass mechanism based on faking
 443         X-Forwarded-For HTTP header. A random country from provided country list
 444         is selected and a random IP belonging to this country is generated. This
 445         IP will be passed as X-Forwarded-For HTTP header in all subsequent
 446         HTTP requests.
 447
 448         This method will be used for initial geo bypass mechanism initialization
 449         during the instance initialization with _GEO_COUNTRIES and
 450         _GEO_IP_BLOCKS.
 451
 452         You may also manually call it from extractor's code if geo bypass
 453         information is not available beforehand (e.g. obtained during
 454         extraction) or due to some other reason. In this case you should pass
 455         this information in geo bypass context passed as first argument. It may
 456         contain following fields:
 457
 458         countries:  List of geo unrestricted countries (similar
 459                     to _GEO_COUNTRIES)
 460         ip_blocks:  List of geo unrestricted IP blocks in CIDR notation
 461                     (similar to _GEO_IP_BLOCKS)
 462
 463         """
 464         if not self._x_forwarded_for_ip:
 465
 466             # Geo bypass mechanism is explicitly disabled by user
 467             if not self._downloader.params.get('geo_bypass', True):
 468                 return
 469
 470             if not geo_bypass_context:
 471                 geo_bypass_context = {}
 472
 473             # Backward compatibility: previously _initialize_geo_bypass
 474             # expected a list of countries, some 3rd party code may still use
 475             # it this way
 476             if isinstance(geo_bypass_context, (list, tuple)):
 477                 geo_bypass_context = {
 478                     'countries': geo_bypass_context,
 479                 }
 480
 481             # The whole point of geo bypass mechanism is to fake IP
 482             # as X-Forwarded-For HTTP header based on some IP block or
 483             # country code.
 484
 485             # Path 1: bypassing based on IP block in CIDR notation
 486
 487             # Explicit IP block specified by user, use it right away
 488             # regardless of whether extractor is geo bypassable or not
 489             ip_block = self._downloader.params.get('geo_bypass_ip_block', None)
 490
 491             # Otherwise use random IP block from geo bypass context but only
 492             # if extractor is known as geo bypassable
 493             if not ip_block:
 494                 ip_blocks = geo_bypass_context.get('ip_blocks')
 495                 if self._GEO_BYPASS and ip_blocks:
 496                     ip_block = random.choice(ip_blocks)
 497
 498             if ip_block:
 499                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block)
 500                 if self._downloader.params.get('verbose', False):
 501                     self._downloader.to_screen(
 502                         '[debug] Using fake IP %s as X-Forwarded-For.'
 503                         % self._x_forwarded_for_ip)
 504                 return
 505
 506             # Path 2: bypassing based on country code
 507
 508             # Explicit country code specified by user, use it right away
 509             # regardless of whether extractor is geo bypassable or not
 510             country = self._downloader.params.get('geo_bypass_country', None)
 511
 512             # Otherwise use random country code from geo bypass context but
 513             # only if extractor is known as geo bypassable
 514             if not country:
 515                 countries = geo_bypass_context.get('countries')
 516                 if self._GEO_BYPASS and countries:
 517                     country = random.choice(countries)
 518
 519             if country:
 520                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country)
 521                 if self._downloader.params.get('verbose', False):
 522                     self._downloader.to_screen(
 523                         '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
 524                         % (self._x_forwarded_for_ip, country.upper()))
 525
 526     def extract(self, url):
 527         """Extracts URL information and returns it in list of dicts."""
 528         try:
 529             for _ in range(2):
 530                 try:
 531                     self.initialize()
 532                     ie_result = self._real_extract(url)
 533                     if self._x_forwarded_for_ip:
 534                         ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
 535                     return ie_result
 536                 except GeoRestrictedError as e:
 537                     if self.__maybe_fake_ip_and_retry(e.countries):
 538                         continue
 539                     raise
 540         except ExtractorError:
 541             raise
 542         except compat_http_client.IncompleteRead as e:
 543             raise ExtractorError('A network error has occurred.', cause=e, expected=True)
 544         except (KeyError, StopIteration) as e:
 545             raise ExtractorError('An extractor error has occurred.', cause=e)
 546
 547     def __maybe_fake_ip_and_retry(self, countries):
 548         if (not self._downloader.params.get('geo_bypass_country', None)
 549                 and self._GEO_BYPASS
 550                 and self._downloader.params.get('geo_bypass', True)
 551                 and not self._x_forwarded_for_ip
 552                 and countries):
 553             country_code = random.choice(countries)
 554             self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
 555             if self._x_forwarded_for_ip:
 556                 self.report_warning(
 557                     'Video is geo restricted. Retrying extraction with fake IP %s (%s) as X-Forwarded-For.'
 558                     % (self._x_forwarded_for_ip, country_code.upper()))
 559                 return True
 560         return False
 561
 562     def set_downloader(self, downloader):
 563         """Sets the downloader for this IE."""
 564         self._downloader = downloader
 565
 566     def _real_initialize(self):
 567         """Real initialization process. Redefine in subclasses."""
 568         pass
 569
 570     def _real_extract(self, url):
 571         """Real extraction process. Redefine in subclasses."""
 572         pass
 573
 574     @classmethod
 575     def ie_key(cls):
 576         """A string for getting the InfoExtractor with get_info_extractor"""
 577         return compat_str(cls.__name__[:-2])
 578
 579     @property
 580     def IE_NAME(self):
 581         return compat_str(type(self).__name__[:-2])
 582
 583     @staticmethod
 584     def __can_accept_status_code(err, expected_status):
 585         assert isinstance(err, compat_urllib_error.HTTPError)
 586         if expected_status is None:
 587             return False
 588         if isinstance(expected_status, compat_integer_types):
 589             return err.code == expected_status
 590         elif isinstance(expected_status, (list, tuple)):
 591             return err.code in expected_status
 592         elif callable(expected_status):
 593             return expected_status(err.code) is True
 594         else:
 595             assert False
 596
 597     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}, expected_status=None):
 598         """
 599         Return the response handle.
 600
 601         See _download_webpage docstring for arguments specification.
 602         """
 603         if note is None:
 604             self.report_download_webpage(video_id)
 605         elif note is not False:
 606             if video_id is None:
 607                 self.to_screen('%s' % (note,))
 608             else:
 609                 self.to_screen('%s: %s' % (video_id, note))
 610
 611         # Some sites check X-Forwarded-For HTTP header in order to figure out
 612         # the origin of the client behind proxy. This allows bypassing geo
 613         # restriction by faking this header's value to IP that belongs to some
 614         # geo unrestricted country. We will do so once we encounter any
 615         # geo restriction error.
 616         if self._x_forwarded_for_ip:
 617             if 'X-Forwarded-For' not in headers:
 618                 headers['X-Forwarded-For'] = self._x_forwarded_for_ip
 619
 620         if isinstance(url_or_request, compat_urllib_request.Request):
 621             url_or_request = update_Request(
 622                 url_or_request, data=data, headers=headers, query=query)
 623         else:
 624             if query:
 625                 url_or_request = update_url_query(url_or_request, query)
 626             if data is not None or headers:
 627                 url_or_request = sanitized_Request(url_or_request, data, headers)
 628         exceptions = [compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error]
 629         if hasattr(ssl, 'CertificateError'):
 630             exceptions.append(ssl.CertificateError)
 631         try:
 632             return self._downloader.urlopen(url_or_request)
 633         except tuple(exceptions) as err:
 634             if isinstance(err, compat_urllib_error.HTTPError):
 635                 if self.__can_accept_status_code(err, expected_status):
 636                     # Retain reference to error to prevent file object from
 637                     # being closed before it can be read. Works around the
 638                     # effects of <https://bugs.python.org/issue15002>
 639                     # introduced in Python 3.4.1.
 640                     err.fp._error = err
 641                     return err.fp
 642
 643             if errnote is False:
 644                 return False
 645             if errnote is None:
 646                 errnote = 'Unable to download webpage'
 647
 648             errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
 649             if fatal:
 650                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 651             else:
 652                 self._downloader.report_warning(errmsg)
 653                 return False
 654
 655     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
 656         """
 657         Return a tuple (page content as string, URL handle).
 658
 659         See _download_webpage docstring for arguments specification.
 660         """
 661         # Strip hashes from the URL (#1038)
 662         if isinstance(url_or_request, (compat_str, str)):
 663             url_or_request = url_or_request.partition('#')[0]
 664
 665         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query, expected_status=expected_status)
 666         if urlh is False:
 667             assert not fatal
 668             return False
 669         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 670         return (content, urlh)
 671
 672     @staticmethod
 673     def _guess_encoding_from_content(content_type, webpage_bytes):
 674         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 675         if m:
 676             encoding = m.group(1)
 677         else:
 678             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 679                           webpage_bytes[:1024])
 680             if m:
 681                 encoding = m.group(1).decode('ascii')
 682             elif webpage_bytes.startswith(b'\xff\xfe'):
 683                 encoding = 'utf-16'
 684             else:
 685                 encoding = 'utf-8'
 686
 687         return encoding
 688
 689     def __check_blocked(self, content):
 690         first_block = content[:512]
 691         if ('<title>Access to this site is blocked</title>' in content
 692                 and 'Websense' in first_block):
 693             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
 694             blocked_iframe = self._html_search_regex(
 695                 r'<iframe src="([^"]+)"', content,
 696                 'Websense information URL', default=None)
 697             if blocked_iframe:
 698                 msg += ' Visit %s for more details' % blocked_iframe
 699             raise ExtractorError(msg, expected=True)
 700         if '<title>The URL you requested has been blocked</title>' in first_block:
 701             msg = (
 702                 'Access to this webpage has been blocked by Indian censorship. '
 703                 'Use a VPN or proxy server (with --proxy) to route around it.')
 704             block_msg = self._html_search_regex(
 705                 r'</h1><p>(.*?)</p>',
 706                 content, 'block message', default=None)
 707             if block_msg:
 708                 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
 709             raise ExtractorError(msg, expected=True)
 710         if ('<title>TTK :: Доступ к ресурсу ограничен</title>' in content
 711                 and 'blocklist.rkn.gov.ru' in content):
 712             raise ExtractorError(
 713                 'Access to this webpage has been blocked by decision of the Russian government. '
 714                 'Visit http://blocklist.rkn.gov.ru/ for a block reason.',
 715                 expected=True)
 716
 717     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
 718         content_type = urlh.headers.get('Content-Type', '')
 719         webpage_bytes = urlh.read()
 720         if prefix is not None:
 721             webpage_bytes = prefix + webpage_bytes
 722         if not encoding:
 723             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
 724         if self._downloader.params.get('dump_intermediate_pages', False):
 725             self.to_screen('Dumping request to ' + urlh.geturl())
 726             dump = base64.b64encode(webpage_bytes).decode('ascii')
 727             self._downloader.to_screen(dump)
 728         if self._downloader.params.get('write_pages', False):
 729             basen = '%s_%s' % (video_id, urlh.geturl())
 730             if len(basen) > 240:
 731                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
 732                 basen = basen[:240 - len(h)] + h
 733             raw_filename = basen + '.dump'
 734             filename = sanitize_filename(raw_filename, restricted=True)
 735             self.to_screen('Saving request to ' + filename)
 736             # Working around MAX_PATH limitation on Windows (see
 737             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
 738             if compat_os_name == 'nt':
 739                 absfilepath = os.path.abspath(filename)
 740                 if len(absfilepath) > 259:
 741                     filename = '\\\\?\\' + absfilepath
 742             with open(filename, 'wb') as outf:
 743                 outf.write(webpage_bytes)
 744
 745         try:
 746             content = webpage_bytes.decode(encoding, 'replace')
 747         except LookupError:
 748             content = webpage_bytes.decode('utf-8', 'replace')
 749
 750         self.__check_blocked(content)
 751
 752         return content
 753
 754     def _download_webpage(
 755             self, url_or_request, video_id, note=None, errnote=None,
 756             fatal=True, tries=1, timeout=5, encoding=None, data=None,
 757             headers={}, query={}, expected_status=None):
 758         """
 759         Return the data of the page as a string.
 760
 761         Arguments:
 762         url_or_request -- plain text URL as a string or
 763             a compat_urllib_request.Requestobject
 764         video_id -- Video/playlist/item identifier (string)
 765
 766         Keyword arguments:
 767         note -- note printed before downloading (string)
 768         errnote -- note printed in case of an error (string)
 769         fatal -- flag denoting whether error should be considered fatal,
 770             i.e. whether it should cause ExtractionError to be raised,
 771             otherwise a warning will be reported and extraction continued
 772         tries -- number of tries
 773         timeout -- sleep interval between tries
 774         encoding -- encoding for a page content decoding, guessed automatically
 775             when not explicitly specified
 776         data -- POST data (bytes)
 777         headers -- HTTP headers (dict)
 778         query -- URL query (dict)
 779         expected_status -- allows to accept failed HTTP requests (non 2xx
 780             status code) by explicitly specifying a set of accepted status
 781             codes. Can be any of the following entities:
 782                 - an integer type specifying an exact failed status code to
 783                   accept
 784                 - a list or a tuple of integer types specifying a list of
 785                   failed status codes to accept
 786                 - a callable accepting an actual failed status code and
 787                   returning True if it should be accepted
 788             Note that this argument does not affect success status codes (2xx)
 789             which are always accepted.
 790         """
 791
 792         success = False
 793         try_count = 0
 794         while success is False:
 795             try:
 796                 res = self._download_webpage_handle(
 797                     url_or_request, video_id, note, errnote, fatal,
 798                     encoding=encoding, data=data, headers=headers, query=query,
 799                     expected_status=expected_status)
 800                 success = True
 801             except compat_http_client.IncompleteRead as e:
 802                 try_count += 1
 803                 if try_count >= tries:
 804                     raise e
 805                 self._sleep(timeout, video_id)
 806         if res is False:
 807             return res
 808         else:
 809             content, _ = res
 810             return content
 811
 812     def _download_xml_handle(
 813             self, url_or_request, video_id, note='Downloading XML',
 814             errnote='Unable to download XML', transform_source=None,
 815             fatal=True, encoding=None, data=None, headers={}, query={},
 816             expected_status=None):
 817         """
 818         Return a tuple (xml as an compat_etree_Element, URL handle).
 819
 820         See _download_webpage docstring for arguments specification.
 821         """
 822         res = self._download_webpage_handle(
 823             url_or_request, video_id, note, errnote, fatal=fatal,
 824             encoding=encoding, data=data, headers=headers, query=query,
 825             expected_status=expected_status)
 826         if res is False:
 827             return res
 828         xml_string, urlh = res
 829         return self._parse_xml(
 830             xml_string, video_id, transform_source=transform_source,
 831             fatal=fatal), urlh
 832
 833     def _download_xml(
 834             self, url_or_request, video_id,
 835             note='Downloading XML', errnote='Unable to download XML',
 836             transform_source=None, fatal=True, encoding=None,
 837             data=None, headers={}, query={}, expected_status=None):
 838         """
 839         Return the xml as an compat_etree_Element.
 840
 841         See _download_webpage docstring for arguments specification.
 842         """
 843         res = self._download_xml_handle(
 844             url_or_request, video_id, note=note, errnote=errnote,
 845             transform_source=transform_source, fatal=fatal, encoding=encoding,
 846             data=data, headers=headers, query=query,
 847             expected_status=expected_status)
 848         return res if res is False else res[0]
 849
 850     def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
 851         if transform_source:
 852             xml_string = transform_source(xml_string)
 853         try:
 854             return compat_etree_fromstring(xml_string.encode('utf-8'))
 855         except compat_xml_parse_error as ve:
 856             errmsg = '%s: Failed to parse XML ' % video_id
 857             if fatal:
 858                 raise ExtractorError(errmsg, cause=ve)
 859             else:
 860                 self.report_warning(errmsg + str(ve))
 861
 862     def _download_json_handle(
 863             self, url_or_request, video_id, note='Downloading JSON metadata',
 864             errnote='Unable to download JSON metadata', transform_source=None,
 865             fatal=True, encoding=None, data=None, headers={}, query={},
 866             expected_status=None):
 867         """
 868         Return a tuple (JSON object, URL handle).
 869
 870         See _download_webpage docstring for arguments specification.
 871         """
 872         res = self._download_webpage_handle(
 873             url_or_request, video_id, note, errnote, fatal=fatal,
 874             encoding=encoding, data=data, headers=headers, query=query,
 875             expected_status=expected_status)
 876         if res is False:
 877             return res
 878         json_string, urlh = res
 879         return self._parse_json(
 880             json_string, video_id, transform_source=transform_source,
 881             fatal=fatal), urlh
 882
 883     def _download_json(
 884             self, url_or_request, video_id, note='Downloading JSON metadata',
 885             errnote='Unable to download JSON metadata', transform_source=None,
 886             fatal=True, encoding=None, data=None, headers={}, query={},
 887             expected_status=None):
 888         """
 889         Return the JSON object as a dict.
 890
 891         See _download_webpage docstring for arguments specification.
 892         """
 893         res = self._download_json_handle(
 894             url_or_request, video_id, note=note, errnote=errnote,
 895             transform_source=transform_source, fatal=fatal, encoding=encoding,
 896             data=data, headers=headers, query=query,
 897             expected_status=expected_status)
 898         return res if res is False else res[0]
 899
 900     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
 901         if transform_source:
 902             json_string = transform_source(json_string)
 903         try:
 904             return json.loads(json_string)
 905         except ValueError as ve:
 906             errmsg = '%s: Failed to parse JSON ' % video_id
 907             if fatal:
 908                 raise ExtractorError(errmsg, cause=ve)
 909             else:
 910                 self.report_warning(errmsg + str(ve))
 911
 912     def report_warning(self, msg, video_id=None):
 913         idstr = '' if video_id is None else '%s: ' % video_id
 914         self._downloader.report_warning(
 915             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
 916
 917     def to_screen(self, msg):
 918         """Print msg to screen, prefixing it with '[ie_name]'"""
 919         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
 920
 921     def report_extraction(self, id_or_name):
 922         """Report information extraction."""
 923         self.to_screen('%s: Extracting information' % id_or_name)
 924
 925     def report_download_webpage(self, video_id):
 926         """Report webpage download."""
 927         self.to_screen('%s: Downloading webpage' % video_id)
 928
 929     def report_age_confirmation(self):
 930         """Report attempt to confirm age."""
 931         self.to_screen('Confirming age')
 932
 933     def report_login(self):
 934         """Report attempt to log in."""
 935         self.to_screen('Logging in')
 936
 937     @staticmethod
 938     def raise_login_required(msg='This video is only available for registered users'):
 939         raise ExtractorError(
 940             '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
 941             expected=True)
 942
 943     @staticmethod
 944     def raise_geo_restricted(msg='This video is not available from your location due to geo restriction', countries=None):
 945         raise GeoRestrictedError(msg, countries=countries)
 946
 947     # Methods for following #608
 948     @staticmethod
 949     def url_result(url, ie=None, video_id=None, video_title=None):
 950         """Returns a URL that points to a page that should be processed"""
 951         # TODO: ie should be the class used for getting the info
 952         video_info = {'_type': 'url',
 953                       'url': url,
 954                       'ie_key': ie}
 955         if video_id is not None:
 956             video_info['id'] = video_id
 957         if video_title is not None:
 958             video_info['title'] = video_title
 959         return video_info
 960
 961     def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None):
 962         urls = orderedSet(
 963             self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
 964             for m in matches)
 965         return self.playlist_result(
 966             urls, playlist_id=playlist_id, playlist_title=playlist_title)
 967
 968     @staticmethod
 969     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
 970         """Returns a playlist"""
 971         video_info = {'_type': 'playlist',
 972                       'entries': entries}
 973         if playlist_id:
 974             video_info['id'] = playlist_id
 975         if playlist_title:
 976             video_info['title'] = playlist_title
 977         if playlist_description:
 978             video_info['description'] = playlist_description
 979         return video_info
 980
 981     def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
 982         """
 983         Perform a regex search on the given string, using a single or a list of
 984         patterns returning the first matching group.
 985         In case of failure return a default value or raise a WARNING or a
 986         RegexNotFoundError, depending on fatal, specifying the field name.
 987         """
 988         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 989             mobj = re.search(pattern, string, flags)
 990         else:
 991             for p in pattern:
 992                 mobj = re.search(p, string, flags)
 993                 if mobj:
 994                     break
 995
 996         if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
 997             _name = '\033[0;34m%s\033[0m' % name
 998         else:
 999             _name = name
1000
1001         if mobj:
1002             if group is None:
1003                 # return the first matching group
1004                 return next(g for g in mobj.groups() if g is not None)
1005             else:
1006                 return mobj.group(group)
1007         elif default is not NO_DEFAULT:
1008             return default
1009         elif fatal:
1010             raise RegexNotFoundError('Unable to extract %s' % _name)
1011         else:
1012             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
1013             return None
1014
1015     def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
1016         """
1017         Like _search_regex, but strips HTML tags and unescapes entities.
1018         """
1019         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
1020         if res:
1021             return clean_html(res).strip()
1022         else:
1023             return res
1024
1025     def _get_netrc_login_info(self, netrc_machine=None):
1026         username = None
1027         password = None
1028         netrc_machine = netrc_machine or self._NETRC_MACHINE
1029
1030         if self._downloader.params.get('usenetrc', False):
1031             try:
1032                 info = netrc.netrc().authenticators(netrc_machine)
1033                 if info is not None:
1034                     username = info[0]
1035                     password = info[2]
1036                 else:
1037                     raise netrc.NetrcParseError(
1038                         'No authenticators for %s' % netrc_machine)
1039             except (IOError, netrc.NetrcParseError) as err:
1040                 self._downloader.report_warning(
1041                     'parsing .netrc: %s' % error_to_compat_str(err))
1042
1043         return username, password
1044
1045     def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
1046         """
1047         Get the login info as (username, password)
1048         First look for the manually specified credentials using username_option
1049         and password_option as keys in params dictionary. If no such credentials
1050         available look in the netrc file using the netrc_machine or _NETRC_MACHINE
1051         value.
1052         If there's no info available, return (None, None)
1053         """
1054         if self._downloader is None:
1055             return (None, None)
1056
1057         downloader_params = self._downloader.params
1058
1059         # Attempt to use provided username and password or .netrc data
1060         if downloader_params.get(username_option) is not None:
1061             username = downloader_params[username_option]
1062             password = downloader_params[password_option]
1063         else:
1064             username, password = self._get_netrc_login_info(netrc_machine)
1065
1066         return username, password
1067
1068     def _get_tfa_info(self, note='two-factor verification code'):
1069         """
1070         Get the two-factor authentication info
1071         TODO - asking the user will be required for sms/phone verify
1072         currently just uses the command line option
1073         If there's no info available, return None
1074         """
1075         if self._downloader is None:
1076             return None
1077         downloader_params = self._downloader.params
1078
1079         if downloader_params.get('twofactor') is not None:
1080             return downloader_params['twofactor']
1081
1082         return compat_getpass('Type %s and press [Return]: ' % note)
1083
1084     # Helper functions for extracting OpenGraph info
1085     @staticmethod
1086     def _og_regexes(prop):
1087         content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
1088         property_re = (r'(?:name|property)=(?:\'og[:-]%(prop)s\'|"og[:-]%(prop)s"|\s*og[:-]%(prop)s\b)'
1089                        % {'prop': re.escape(prop)})
1090         template = r'<meta[^>]+?%s[^>]+?%s'
1091         return [
1092             template % (property_re, content_re),
1093             template % (content_re, property_re),
1094         ]
1095
1096     @staticmethod
1097     def _meta_regex(prop):
1098         return r'''(?isx)<meta
1099                     (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
1100                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
1101
1102     def _og_search_property(self, prop, html, name=None, **kargs):
1103         if not isinstance(prop, (list, tuple)):
1104             prop = [prop]
1105         if name is None:
1106             name = 'OpenGraph %s' % prop[0]
1107         og_regexes = []
1108         for p in prop:
1109             og_regexes.extend(self._og_regexes(p))
1110         escaped = self._search_regex(og_regexes, html, name, flags=re.DOTALL, **kargs)
1111         if escaped is None:
1112             return None
1113         return unescapeHTML(escaped)
1114
1115     def _og_search_thumbnail(self, html, **kargs):
1116         return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
1117
1118     def _og_search_description(self, html, **kargs):
1119         return self._og_search_property('description', html, fatal=False, **kargs)
1120
1121     def _og_search_title(self, html, **kargs):
1122         return self._og_search_property('title', html, **kargs)
1123
1124     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
1125         regexes = self._og_regexes('video') + self._og_regexes('video:url')
1126         if secure:
1127             regexes = self._og_regexes('video:secure_url') + regexes
1128         return self._html_search_regex(regexes, html, name, **kargs)
1129
1130     def _og_search_url(self, html, **kargs):
1131         return self._og_search_property('url', html, **kargs)
1132
1133     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
1134         if not isinstance(name, (list, tuple)):
1135             name = [name]
1136         if display_name is None:
1137             display_name = name[0]
1138         return self._html_search_regex(
1139             [self._meta_regex(n) for n in name],
1140             html, display_name, fatal=fatal, group='content', **kwargs)
1141
1142     def _dc_search_uploader(self, html):
1143         return self._html_search_meta('dc.creator', html, 'uploader')
1144
1145     def _rta_search(self, html):
1146         # See http://www.rtalabel.org/index.php?content=howtofaq#single
1147         if re.search(r'(?ix)<meta\s+name="rating"\s+'
1148                      r'     content="RTA-5042-1996-1400-1577-RTA"',
1149                      html):
1150             return 18
1151         return 0
1152
1153     def _media_rating_search(self, html):
1154         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
1155         rating = self._html_search_meta('rating', html)
1156
1157         if not rating:
1158             return None
1159
1160         RATING_TABLE = {
1161             'safe for kids': 0,
1162             'general': 8,
1163             '14 years': 14,
1164             'mature': 17,
1165             'restricted': 19,
1166         }
1167         return RATING_TABLE.get(rating.lower())
1168
1169     def _family_friendly_search(self, html):
1170         # See http://schema.org/VideoObject
1171         family_friendly = self._html_search_meta(
1172             'isFamilyFriendly', html, default=None)
1173
1174         if not family_friendly:
1175             return None
1176
1177         RATING_TABLE = {
1178             '1': 0,
1179             'true': 0,
1180             '0': 18,
1181             'false': 18,
1182         }
1183         return RATING_TABLE.get(family_friendly.lower())
1184
1185     def _twitter_search_player(self, html):
1186         return self._html_search_meta('twitter:player', html,
1187                                       'twitter card player')
1188
1189     def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
1190         json_ld_list = list(re.finditer(JSON_LD_RE, html))
1191         default = kwargs.get('default', NO_DEFAULT)
1192         # JSON-LD may be malformed and thus `fatal` should be respected.
1193         # At the same time `default` may be passed that assumes `fatal=False`
1194         # for _search_regex. Let's simulate the same behavior here as well.
1195         fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
1196         json_ld = []
1197         for mobj in json_ld_list:
1198             json_ld_item = self._parse_json(
1199                 mobj.group('json_ld'), video_id, fatal=fatal)
1200             if not json_ld_item:
1201                 continue
1202             if isinstance(json_ld_item, dict):
1203                 json_ld.append(json_ld_item)
1204             elif isinstance(json_ld_item, (list, tuple)):
1205                 json_ld.extend(json_ld_item)
1206         if json_ld:
1207             json_ld = self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)
1208         if json_ld:
1209             return json_ld
1210         if default is not NO_DEFAULT:
1211             return default
1212         elif fatal:
1213             raise RegexNotFoundError('Unable to extract JSON-LD')
1214         else:
1215             self._downloader.report_warning('unable to extract JSON-LD %s' % bug_reports_message())
1216             return {}
1217
1218     def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
1219         if isinstance(json_ld, compat_str):
1220             json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
1221         if not json_ld:
1222             return {}
1223         info = {}
1224         if not isinstance(json_ld, (list, tuple, dict)):
1225             return info
1226         if isinstance(json_ld, dict):
1227             json_ld = [json_ld]
1228
1229         INTERACTION_TYPE_MAP = {
1230             'CommentAction': 'comment',
1231             'AgreeAction': 'like',
1232             'DisagreeAction': 'dislike',
1233             'LikeAction': 'like',
1234             'DislikeAction': 'dislike',
1235             'ListenAction': 'view',
1236             'WatchAction': 'view',
1237             'ViewAction': 'view',
1238         }
1239
1240         def extract_interaction_statistic(e):
1241             interaction_statistic = e.get('interactionStatistic')
1242             if not isinstance(interaction_statistic, list):
1243                 return
1244             for is_e in interaction_statistic:
1245                 if not isinstance(is_e, dict):
1246                     continue
1247                 if is_e.get('@type') != 'InteractionCounter':
1248                     continue
1249                 interaction_type = is_e.get('interactionType')
1250                 if not isinstance(interaction_type, compat_str):
1251                     continue
1252                 # For interaction count some sites provide string instead of
1253                 # an integer (as per spec) with non digit characters (e.g. ",")
1254                 # so extracting count with more relaxed str_to_int
1255                 interaction_count = str_to_int(is_e.get('userInteractionCount'))
1256                 if interaction_count is None:
1257                     continue
1258                 count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1])
1259                 if not count_kind:
1260                     continue
1261                 count_key = '%s_count' % count_kind
1262                 if info.get(count_key) is not None:
1263                     continue
1264                 info[count_key] = interaction_count
1265
1266         def extract_video_object(e):
1267             assert e['@type'] == 'VideoObject'
1268             info.update({
1269                 'url': url_or_none(e.get('contentUrl')),
1270                 'title': unescapeHTML(e.get('name')),
1271                 'description': unescapeHTML(e.get('description')),
1272                 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
1273                 'duration': parse_duration(e.get('duration')),
1274                 'timestamp': unified_timestamp(e.get('uploadDate')),
1275                 'uploader': str_or_none(e.get('author')),
1276                 'filesize': float_or_none(e.get('contentSize')),
1277                 'tbr': int_or_none(e.get('bitrate')),
1278                 'width': int_or_none(e.get('width')),
1279                 'height': int_or_none(e.get('height')),
1280                 'view_count': int_or_none(e.get('interactionCount')),
1281             })
1282             extract_interaction_statistic(e)
1283
1284         for e in json_ld:
1285             if '@context' in e:
1286                 item_type = e.get('@type')
1287                 if expected_type is not None and expected_type != item_type:
1288                     continue
1289                 if item_type in ('TVEpisode', 'Episode'):
1290                     episode_name = unescapeHTML(e.get('name'))
1291                     info.update({
1292                         'episode': episode_name,
1293                         'episode_number': int_or_none(e.get('episodeNumber')),
1294                         'description': unescapeHTML(e.get('description')),
1295                     })
1296                     if not info.get('title') and episode_name:
1297                         info['title'] = episode_name
1298                     part_of_season = e.get('partOfSeason')
1299                     if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
1300                         info.update({
1301                             'season': unescapeHTML(part_of_season.get('name')),
1302                             'season_number': int_or_none(part_of_season.get('seasonNumber')),
1303                         })
1304                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
1305                     if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
1306                         info['series'] = unescapeHTML(part_of_series.get('name'))
1307                 elif item_type == 'Movie':
1308                     info.update({
1309                         'title': unescapeHTML(e.get('name')),
1310                         'description': unescapeHTML(e.get('description')),
1311                         'duration': parse_duration(e.get('duration')),
1312                         'timestamp': unified_timestamp(e.get('dateCreated')),
1313                     })
1314                 elif item_type in ('Article', 'NewsArticle'):
1315                     info.update({
1316                         'timestamp': parse_iso8601(e.get('datePublished')),
1317                         'title': unescapeHTML(e.get('headline')),
1318                         'description': unescapeHTML(e.get('articleBody')),
1319                     })
1320                 elif item_type == 'VideoObject':
1321                     extract_video_object(e)
1322                     if expected_type is None:
1323                         continue
1324                     else:
1325                         break
1326                 video = e.get('video')
1327                 if isinstance(video, dict) and video.get('@type') == 'VideoObject':
1328                     extract_video_object(video)
1329                 if expected_type is None:
1330                     continue
1331                 else:
1332                     break
1333         return dict((k, v) for k, v in info.items() if v is not None)
1334
1335     @staticmethod
1336     def _hidden_inputs(html):
1337         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
1338         hidden_inputs = {}
1339         for input in re.findall(r'(?i)(<input[^>]+>)', html):
1340             attrs = extract_attributes(input)
1341             if not input:
1342                 continue
1343             if attrs.get('type') not in ('hidden', 'submit'):
1344                 continue
1345             name = attrs.get('name') or attrs.get('id')
1346             value = attrs.get('value')
1347             if name and value is not None:
1348                 hidden_inputs[name] = value
1349         return hidden_inputs
1350
1351     def _form_hidden_inputs(self, form_id, html):
1352         form = self._search_regex(
1353             r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
1354             html, '%s form' % form_id, group='form')
1355         return self._hidden_inputs(form)
1356
1357     def _sort_formats(self, formats, field_preference=None):
1358         if not formats:
1359             raise ExtractorError('No video formats found')
1360
1361         for f in formats:
1362             # Automatically determine tbr when missing based on abr and vbr (improves
1363             # formats sorting in some cases)
1364             if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
1365                 f['tbr'] = f['abr'] + f['vbr']
1366
1367         def _formats_key(f):
1368             # TODO remove the following workaround
1369             from ..utils import determine_ext
1370             if not f.get('ext') and 'url' in f:
1371                 f['ext'] = determine_ext(f['url'])
1372
1373             if isinstance(field_preference, (list, tuple)):
1374                 return tuple(
1375                     f.get(field)
1376                     if f.get(field) is not None
1377                     else ('' if field == 'format_id' else -1)
1378                     for field in field_preference)
1379
1380             preference = f.get('preference')
1381             if preference is None:
1382                 preference = 0
1383                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
1384                     preference -= 0.5
1385
1386             protocol = f.get('protocol') or determine_protocol(f)
1387             proto_preference = 0 if protocol in ['http', 'https'] else (-0.5 if protocol == 'rtsp' else -0.1)
1388
1389             if f.get('vcodec') == 'none':  # audio only
1390                 preference -= 50
1391                 if self._downloader.params.get('prefer_free_formats'):
1392                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
1393                 else:
1394                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
1395                 ext_preference = 0
1396                 try:
1397                     audio_ext_preference = ORDER.index(f['ext'])
1398                 except ValueError:
1399                     audio_ext_preference = -1
1400             else:
1401                 if f.get('acodec') == 'none':  # video only
1402                     preference -= 40
1403                 if self._downloader.params.get('prefer_free_formats'):
1404                     ORDER = ['flv', 'mp4', 'webm']
1405                 else:
1406                     ORDER = ['webm', 'flv', 'mp4']
1407                 try:
1408                     ext_preference = ORDER.index(f['ext'])
1409                 except ValueError:
1410                     ext_preference = -1
1411                 audio_ext_preference = 0
1412
1413             return (
1414                 preference,
1415                 f.get('language_preference') if f.get('language_preference') is not None else -1,
1416                 f.get('quality') if f.get('quality') is not None else -1,
1417                 f.get('tbr') if f.get('tbr') is not None else -1,
1418                 f.get('filesize') if f.get('filesize') is not None else -1,
1419                 f.get('vbr') if f.get('vbr') is not None else -1,
1420                 f.get('height') if f.get('height') is not None else -1,
1421                 f.get('width') if f.get('width') is not None else -1,
1422                 proto_preference,
1423                 ext_preference,
1424                 f.get('abr') if f.get('abr') is not None else -1,
1425                 audio_ext_preference,
1426                 f.get('fps') if f.get('fps') is not None else -1,
1427                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
1428                 f.get('source_preference') if f.get('source_preference') is not None else -1,
1429                 f.get('format_id') if f.get('format_id') is not None else '',
1430             )
1431         formats.sort(key=_formats_key)
1432
1433     def _check_formats(self, formats, video_id):
1434         if formats:
1435             formats[:] = filter(
1436                 lambda f: self._is_valid_url(
1437                     f['url'], video_id,
1438                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
1439                 formats)
1440
1441     @staticmethod
1442     def _remove_duplicate_formats(formats):
1443         format_urls = set()
1444         unique_formats = []
1445         for f in formats:
1446             if f['url'] not in format_urls:
1447                 format_urls.add(f['url'])
1448                 unique_formats.append(f)
1449         formats[:] = unique_formats
1450
1451     def _is_valid_url(self, url, video_id, item='video', headers={}):
1452         url = self._proto_relative_url(url, scheme='http:')
1453         # For now assume non HTTP(S) URLs always valid
1454         if not (url.startswith('http://') or url.startswith('https://')):
1455             return True
1456         try:
1457             self._request_webpage(url, video_id, 'Checking %s URL' % item, headers=headers)
1458             return True
1459         except ExtractorError:
1460             self.to_screen(
1461                 '%s: %s URL is invalid, skipping' % (video_id, item))
1462             return False
1463
1464     def http_scheme(self):
1465         """ Either "http:" or "https:", depending on the user's preferences """
1466         return (
1467             'http:'
1468             if self._downloader.params.get('prefer_insecure', False)
1469             else 'https:')
1470
1471     def _proto_relative_url(self, url, scheme=None):
1472         if url is None:
1473             return url
1474         if url.startswith('//'):
1475             if scheme is None:
1476                 scheme = self.http_scheme()
1477             return scheme + url
1478         else:
1479             return url
1480
1481     def _sleep(self, timeout, video_id, msg_template=None):
1482         if msg_template is None:
1483             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
1484         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
1485         self.to_screen(msg)
1486         time.sleep(timeout)
1487
1488     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
1489                              transform_source=lambda s: fix_xml_ampersands(s).strip(),
1490                              fatal=True, m3u8_id=None, data=None, headers={}, query={}):
1491         manifest = self._download_xml(
1492             manifest_url, video_id, 'Downloading f4m manifest',
1493             'Unable to download f4m manifest',
1494             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
1495             # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244)
1496             transform_source=transform_source,
1497             fatal=fatal, data=data, headers=headers, query=query)
1498
1499         if manifest is False:
1500             return []
1501
1502         return self._parse_f4m_formats(
1503             manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1504             transform_source=transform_source, fatal=fatal, m3u8_id=m3u8_id)
1505
1506     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
1507                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
1508                            fatal=True, m3u8_id=None):
1509         if not isinstance(manifest, compat_etree_Element) and not fatal:
1510             return []
1511
1512         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
1513         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
1514         if akamai_pv is not None and ';' in akamai_pv.text:
1515             playerVerificationChallenge = akamai_pv.text.split(';')[0]
1516             if playerVerificationChallenge.strip() != '':
1517                 return []
1518
1519         formats = []
1520         manifest_version = '1.0'
1521         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
1522         if not media_nodes:
1523             manifest_version = '2.0'
1524             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
1525         # Remove unsupported DRM protected media from final formats
1526         # rendition (see https://github.com/ytdl-org/youtube-dl/issues/8573).
1527         media_nodes = remove_encrypted_media(media_nodes)
1528         if not media_nodes:
1529             return formats
1530
1531         manifest_base_url = get_base_url(manifest)
1532
1533         bootstrap_info = xpath_element(
1534             manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
1535             'bootstrap info', default=None)
1536
1537         vcodec = None
1538         mime_type = xpath_text(
1539             manifest, ['{http://ns.adobe.com/f4m/1.0}mimeType', '{http://ns.adobe.com/f4m/2.0}mimeType'],
1540             'base URL', default=None)
1541         if mime_type and mime_type.startswith('audio/'):
1542             vcodec = 'none'
1543
1544         for i, media_el in enumerate(media_nodes):
1545             tbr = int_or_none(media_el.attrib.get('bitrate'))
1546             width = int_or_none(media_el.attrib.get('width'))
1547             height = int_or_none(media_el.attrib.get('height'))
1548             format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
1549             # If <bootstrapInfo> is present, the specified f4m is a
1550             # stream-level manifest, and only set-level manifests may refer to
1551             # external resources.  See section 11.4 and section 4 of F4M spec
1552             if bootstrap_info is None:
1553                 media_url = None
1554                 # @href is introduced in 2.0, see section 11.6 of F4M spec
1555                 if manifest_version == '2.0':
1556                     media_url = media_el.attrib.get('href')
1557                 if media_url is None:
1558                     media_url = media_el.attrib.get('url')
1559                 if not media_url:
1560                     continue
1561                 manifest_url = (
1562                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
1563                     else ((manifest_base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
1564                 # If media_url is itself a f4m manifest do the recursive extraction
1565                 # since bitrates in parent manifest (this one) and media_url manifest
1566                 # may differ leading to inability to resolve the format by requested
1567                 # bitrate in f4m downloader
1568                 ext = determine_ext(manifest_url)
1569                 if ext == 'f4m':
1570                     f4m_formats = self._extract_f4m_formats(
1571                         manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1572                         transform_source=transform_source, fatal=fatal)
1573                     # Sometimes stream-level manifest contains single media entry that
1574                     # does not contain any quality metadata (e.g. http://matchtv.ru/#live-player).
1575                     # At the same time parent's media entry in set-level manifest may
1576                     # contain it. We will copy it from parent in such cases.
1577                     if len(f4m_formats) == 1:
1578                         f = f4m_formats[0]
1579                         f.update({
1580                             'tbr': f.get('tbr') or tbr,
1581                             'width': f.get('width') or width,
1582                             'height': f.get('height') or height,
1583                             'format_id': f.get('format_id') if not tbr else format_id,
1584                             'vcodec': vcodec,
1585                         })
1586                     formats.extend(f4m_formats)
1587                     continue
1588                 elif ext == 'm3u8':
1589                     formats.extend(self._extract_m3u8_formats(
1590                         manifest_url, video_id, 'mp4', preference=preference,
1591                         m3u8_id=m3u8_id, fatal=fatal))
1592                     continue
1593             formats.append({
1594                 'format_id': format_id,
1595                 'url': manifest_url,
1596                 'manifest_url': manifest_url,
1597                 'ext': 'flv' if bootstrap_info is not None else None,
1598                 'protocol': 'f4m',
1599                 'tbr': tbr,
1600                 'width': width,
1601                 'height': height,
1602                 'vcodec': vcodec,
1603                 'preference': preference,
1604             })
1605         return formats
1606
1607     def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, m3u8_id=None):
1608         return {
1609             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
1610             'url': m3u8_url,
1611             'ext': ext,
1612             'protocol': 'm3u8',
1613             'preference': preference - 100 if preference else -100,
1614             'resolution': 'multiple',
1615             'format_note': 'Quality selection URL',
1616         }
1617
1618     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
1619                               entry_protocol='m3u8', preference=None,
1620                               m3u8_id=None, note=None, errnote=None,
1621                               fatal=True, live=False, data=None, headers={},
1622                               query={}):
1623         res = self._download_webpage_handle(
1624             m3u8_url, video_id,
1625             note=note or 'Downloading m3u8 information',
1626             errnote=errnote or 'Failed to download m3u8 information',
1627             fatal=fatal, data=data, headers=headers, query=query)
1628
1629         if res is False:
1630             return []
1631
1632         m3u8_doc, urlh = res
1633         m3u8_url = urlh.geturl()
1634
1635         return self._parse_m3u8_formats(
1636             m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
1637             preference=preference, m3u8_id=m3u8_id, live=live)
1638
1639     def _parse_m3u8_formats(self, m3u8_doc, m3u8_url, ext=None,
1640                             entry_protocol='m3u8', preference=None,
1641                             m3u8_id=None, live=False):
1642         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
1643             return []
1644
1645         if re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc):  # Apple FairPlay
1646             return []
1647
1648         formats = []
1649
1650         format_url = lambda u: (
1651             u
1652             if re.match(r'^https?://', u)
1653             else compat_urlparse.urljoin(m3u8_url, u))
1654
1655         # References:
1656         # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
1657         # 2. https://github.com/ytdl-org/youtube-dl/issues/12211
1658         # 3. https://github.com/ytdl-org/youtube-dl/issues/18923
1659
1660         # We should try extracting formats only from master playlists [1, 4.3.4],
1661         # i.e. playlists that describe available qualities. On the other hand
1662         # media playlists [1, 4.3.3] should be returned as is since they contain
1663         # just the media without qualities renditions.
1664         # Fortunately, master playlist can be easily distinguished from media
1665         # playlist based on particular tags availability. As of [1, 4.3.3, 4.3.4]
1666         # master playlist tags MUST NOT appear in a media playist and vice versa.
1667         # As of [1, 4.3.3.1] #EXT-X-TARGETDURATION tag is REQUIRED for every
1668         # media playlist and MUST NOT appear in master playlist thus we can
1669         # clearly detect media playlist with this criterion.
1670
1671         if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
1672             return [{
1673                 'url': m3u8_url,
1674                 'format_id': m3u8_id,
1675                 'ext': ext,
1676                 'protocol': entry_protocol,
1677                 'preference': preference,
1678             }]
1679
1680         groups = {}
1681         last_stream_inf = {}
1682
1683         def extract_media(x_media_line):
1684             media = parse_m3u8_attributes(x_media_line)
1685             # As per [1, 4.3.4.1] TYPE, GROUP-ID and NAME are REQUIRED
1686             media_type, group_id, name = media.get('TYPE'), media.get('GROUP-ID'), media.get('NAME')
1687             if not (media_type and group_id and name):
1688                 return
1689             groups.setdefault(group_id, []).append(media)
1690             if media_type not in ('VIDEO', 'AUDIO'):
1691                 return
1692             media_url = media.get('URI')
1693             if media_url:
1694                 format_id = []
1695                 for v in (m3u8_id, group_id, name):
1696                     if v:
1697                         format_id.append(v)
1698                 f = {
1699                     'format_id': '-'.join(format_id),
1700                     'url': format_url(media_url),
1701                     'manifest_url': m3u8_url,
1702                     'language': media.get('LANGUAGE'),
1703                     'ext': ext,
1704                     'protocol': entry_protocol,
1705                     'preference': preference,
1706                 }
1707                 if media_type == 'AUDIO':
1708                     f['vcodec'] = 'none'
1709                 formats.append(f)
1710
1711         def build_stream_name():
1712             # Despite specification does not mention NAME attribute for
1713             # EXT-X-STREAM-INF tag it still sometimes may be present (see [1]
1714             # or vidio test in TestInfoExtractor.test_parse_m3u8_formats)
1715             # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
1716             stream_name = last_stream_inf.get('NAME')
1717             if stream_name:
1718                 return stream_name
1719             # If there is no NAME in EXT-X-STREAM-INF it will be obtained
1720             # from corresponding rendition group
1721             stream_group_id = last_stream_inf.get('VIDEO')
1722             if not stream_group_id:
1723                 return
1724             stream_group = groups.get(stream_group_id)
1725             if not stream_group:
1726                 return stream_group_id
1727             rendition = stream_group[0]
1728             return rendition.get('NAME') or stream_group_id
1729
1730         # parse EXT-X-MEDIA tags before EXT-X-STREAM-INF in order to have the
1731         # chance to detect video only formats when EXT-X-STREAM-INF tags
1732         # precede EXT-X-MEDIA tags in HLS manifest such as [3].
1733         for line in m3u8_doc.splitlines():
1734             if line.startswith('#EXT-X-MEDIA:'):
1735                 extract_media(line)
1736
1737         for line in m3u8_doc.splitlines():
1738             if line.startswith('#EXT-X-STREAM-INF:'):
1739                 last_stream_inf = parse_m3u8_attributes(line)
1740             elif line.startswith('#') or not line.strip():
1741                 continue
1742             else:
1743                 tbr = float_or_none(
1744                     last_stream_inf.get('AVERAGE-BANDWIDTH')
1745                     or last_stream_inf.get('BANDWIDTH'), scale=1000)
1746                 format_id = []
1747                 if m3u8_id:
1748                     format_id.append(m3u8_id)
1749                 stream_name = build_stream_name()
1750                 # Bandwidth of live streams may differ over time thus making
1751                 # format_id unpredictable. So it's better to keep provided
1752                 # format_id intact.
1753                 if not live:
1754                     format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
1755                 manifest_url = format_url(line.strip())
1756                 f = {
1757                     'format_id': '-'.join(format_id),
1758                     'url': manifest_url,
1759                     'manifest_url': m3u8_url,
1760                     'tbr': tbr,
1761                     'ext': ext,
1762                     'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
1763                     'protocol': entry_protocol,
1764                     'preference': preference,
1765                 }
1766                 resolution = last_stream_inf.get('RESOLUTION')
1767                 if resolution:
1768                     mobj = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', resolution)
1769                     if mobj:
1770                         f['width'] = int(mobj.group('width'))
1771                         f['height'] = int(mobj.group('height'))
1772                 # Unified Streaming Platform
1773                 mobj = re.search(
1774                     r'audio.*?(?:%3D|=)(\d+)(?:-video.*?(?:%3D|=)(\d+))?', f['url'])
1775                 if mobj:
1776                     abr, vbr = mobj.groups()
1777                     abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
1778                     f.update({
1779                         'vbr': vbr,
1780                         'abr': abr,
1781                     })
1782                 codecs = parse_codecs(last_stream_inf.get('CODECS'))
1783                 f.update(codecs)
1784                 audio_group_id = last_stream_inf.get('AUDIO')
1785                 # As per [1, 4.3.4.1.1] any EXT-X-STREAM-INF tag which
1786                 # references a rendition group MUST have a CODECS attribute.
1787                 # However, this is not always respected, for example, [2]
1788                 # contains EXT-X-STREAM-INF tag which references AUDIO
1789                 # rendition group but does not have CODECS and despite
1790                 # referencing an audio group it represents a complete
1791                 # (with audio and video) format. So, for such cases we will
1792                 # ignore references to rendition groups and treat them
1793                 # as complete formats.
1794                 if audio_group_id and codecs and f.get('vcodec') != 'none':
1795                     audio_group = groups.get(audio_group_id)
1796                     if audio_group and audio_group[0].get('URI'):
1797                         # TODO: update acodec for audio only formats with
1798                         # the same GROUP-ID
1799                         f['acodec'] = 'none'
1800                 formats.append(f)
1801
1802                 # for DailyMotion
1803                 progressive_uri = last_stream_inf.get('PROGRESSIVE-URI')
1804                 if progressive_uri:
1805                     http_f = f.copy()
1806                     del http_f['manifest_url']
1807                     http_f.update({
1808                         'format_id': f['format_id'].replace('hls-', 'http-'),
1809                         'protocol': 'http',
1810                         'url': progressive_uri,
1811                     })
1812                     formats.append(http_f)
1813
1814                 last_stream_inf = {}
1815         return formats
1816
1817     @staticmethod
1818     def _xpath_ns(path, namespace=None):
1819         if not namespace:
1820             return path
1821         out = []
1822         for c in path.split('/'):
1823             if not c or c == '.':
1824                 out.append(c)
1825             else:
1826                 out.append('{%s}%s' % (namespace, c))
1827         return '/'.join(out)
1828
1829     def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
1830         smil = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
1831
1832         if smil is False:
1833             assert not fatal
1834             return []
1835
1836         namespace = self._parse_smil_namespace(smil)
1837
1838         return self._parse_smil_formats(
1839             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1840
1841     def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
1842         smil = self._download_smil(smil_url, video_id, fatal=fatal)
1843         if smil is False:
1844             return {}
1845         return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
1846
1847     def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
1848         return self._download_xml(
1849             smil_url, video_id, 'Downloading SMIL file',
1850             'Unable to download SMIL file', fatal=fatal, transform_source=transform_source)
1851
1852     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
1853         namespace = self._parse_smil_namespace(smil)
1854
1855         formats = self._parse_smil_formats(
1856             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1857         subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
1858
1859         video_id = os.path.splitext(url_basename(smil_url))[0]
1860         title = None
1861         description = None
1862         upload_date = None
1863         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1864             name = meta.attrib.get('name')
1865             content = meta.attrib.get('content')
1866             if not name or not content:
1867                 continue
1868             if not title and name == 'title':
1869                 title = content
1870             elif not description and name in ('description', 'abstract'):
1871                 description = content
1872             elif not upload_date and name == 'date':
1873                 upload_date = unified_strdate(content)
1874
1875         thumbnails = [{
1876             'id': image.get('type'),
1877             'url': image.get('src'),
1878             'width': int_or_none(image.get('width')),
1879             'height': int_or_none(image.get('height')),
1880         } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
1881
1882         return {
1883             'id': video_id,
1884             'title': title or video_id,
1885             'description': description,
1886             'upload_date': upload_date,
1887             'thumbnails': thumbnails,
1888             'formats': formats,
1889             'subtitles': subtitles,
1890         }
1891
1892     def _parse_smil_namespace(self, smil):
1893         return self._search_regex(
1894             r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
1895
1896     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
1897         base = smil_url
1898         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1899             b = meta.get('base') or meta.get('httpBase')
1900             if b:
1901                 base = b
1902                 break
1903
1904         formats = []
1905         rtmp_count = 0
1906         http_count = 0
1907         m3u8_count = 0
1908
1909         srcs = []
1910         media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
1911         for medium in media:
1912             src = medium.get('src')
1913             if not src or src in srcs:
1914                 continue
1915             srcs.append(src)
1916
1917             bitrate = float_or_none(medium.get('system-bitrate') or medium.get('systemBitrate'), 1000)
1918             filesize = int_or_none(medium.get('size') or medium.get('fileSize'))
1919             width = int_or_none(medium.get('width'))
1920             height = int_or_none(medium.get('height'))
1921             proto = medium.get('proto')
1922             ext = medium.get('ext')
1923             src_ext = determine_ext(src)
1924             streamer = medium.get('streamer') or base
1925
1926             if proto == 'rtmp' or streamer.startswith('rtmp'):
1927                 rtmp_count += 1
1928                 formats.append({
1929                     'url': streamer,
1930                     'play_path': src,
1931                     'ext': 'flv',
1932                     'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
1933                     'tbr': bitrate,
1934                     'filesize': filesize,
1935                     'width': width,
1936                     'height': height,
1937                 })
1938                 if transform_rtmp_url:
1939                     streamer, src = transform_rtmp_url(streamer, src)
1940                     formats[-1].update({
1941                         'url': streamer,
1942                         'play_path': src,
1943                     })
1944                 continue
1945
1946             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
1947             src_url = src_url.strip()
1948
1949             if proto == 'm3u8' or src_ext == 'm3u8':
1950                 m3u8_formats = self._extract_m3u8_formats(
1951                     src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
1952                 if len(m3u8_formats) == 1:
1953                     m3u8_count += 1
1954                     m3u8_formats[0].update({
1955                         'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
1956                         'tbr': bitrate,
1957                         'width': width,
1958                         'height': height,
1959                     })
1960                 formats.extend(m3u8_formats)
1961             elif src_ext == 'f4m':
1962                 f4m_url = src_url
1963                 if not f4m_params:
1964                     f4m_params = {
1965                         'hdcore': '3.2.0',
1966                         'plugin': 'flowplayer-3.2.0.1',
1967                     }
1968                 f4m_url += '&' if '?' in f4m_url else '?'
1969                 f4m_url += compat_urllib_parse_urlencode(f4m_params)
1970                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
1971             elif src_ext == 'mpd':
1972                 formats.extend(self._extract_mpd_formats(
1973                     src_url, video_id, mpd_id='dash', fatal=False))
1974             elif re.search(r'\.ism/[Mm]anifest', src_url):
1975                 formats.extend(self._extract_ism_formats(
1976                     src_url, video_id, ism_id='mss', fatal=False))
1977             elif src_url.startswith('http') and self._is_valid_url(src, video_id):
1978                 http_count += 1
1979                 formats.append({
1980                     'url': src_url,
1981                     'ext': ext or src_ext or 'flv',
1982                     'format_id': 'http-%d' % (bitrate or http_count),
1983                     'tbr': bitrate,
1984                     'filesize': filesize,
1985                     'width': width,
1986                     'height': height,
1987                 })
1988
1989         return formats
1990
1991     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
1992         urls = []
1993         subtitles = {}
1994         for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
1995             src = textstream.get('src')
1996             if not src or src in urls:
1997                 continue
1998             urls.append(src)
1999             ext = textstream.get('ext') or mimetype2ext(textstream.get('type')) or determine_ext(src)
2000             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
2001             subtitles.setdefault(lang, []).append({
2002                 'url': src,
2003                 'ext': ext,
2004             })
2005         return subtitles
2006
2007     def _extract_xspf_playlist(self, xspf_url, playlist_id, fatal=True):
2008         xspf = self._download_xml(
2009             xspf_url, playlist_id, 'Downloading xpsf playlist',
2010             'Unable to download xspf manifest', fatal=fatal)
2011         if xspf is False:
2012             return []
2013         return self._parse_xspf(
2014             xspf, playlist_id, xspf_url=xspf_url,
2015             xspf_base_url=base_url(xspf_url))
2016
2017     def _parse_xspf(self, xspf_doc, playlist_id, xspf_url=None, xspf_base_url=None):
2018         NS_MAP = {
2019             'xspf': 'http://xspf.org/ns/0/',
2020             's1': 'http://static.streamone.nl/player/ns/0',
2021         }
2022
2023         entries = []
2024         for track in xspf_doc.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
2025             title = xpath_text(
2026                 track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
2027             description = xpath_text(
2028                 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
2029             thumbnail = xpath_text(
2030                 track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
2031             duration = float_or_none(
2032                 xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
2033
2034             formats = []
2035             for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP)):
2036                 format_url = urljoin(xspf_base_url, location.text)
2037                 if not format_url:
2038                     continue
2039                 formats.append({
2040                     'url': format_url,
2041                     'manifest_url': xspf_url,
2042                     'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
2043                     'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
2044                     'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
2045                 })
2046             self._sort_formats(formats)
2047
2048             entries.append({
2049                 'id': playlist_id,
2050                 'title': title,
2051                 'description': description,
2052                 'thumbnail': thumbnail,
2053                 'duration': duration,
2054                 'formats': formats,
2055             })
2056         return entries
2057
2058     def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}, data=None, headers={}, query={}):
2059         res = self._download_xml_handle(
2060             mpd_url, video_id,
2061             note=note or 'Downloading MPD manifest',
2062             errnote=errnote or 'Failed to download MPD manifest',
2063             fatal=fatal, data=data, headers=headers, query=query)
2064         if res is False:
2065             return []
2066         mpd_doc, urlh = res
2067         if mpd_doc is None:
2068             return []
2069         mpd_base_url = base_url(urlh.geturl())
2070
2071         return self._parse_mpd_formats(
2072             mpd_doc, mpd_id=mpd_id, mpd_base_url=mpd_base_url,
2073             formats_dict=formats_dict, mpd_url=mpd_url)
2074
2075     def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}, mpd_url=None):
2076         """
2077         Parse formats from MPD manifest.
2078         References:
2079          1. MPEG-DASH Standard, ISO/IEC 23009-1:2014(E),
2080             http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip
2081          2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
2082         """
2083         if mpd_doc.get('type') == 'dynamic':
2084             return []
2085
2086         namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
2087
2088         def _add_ns(path):
2089             return self._xpath_ns(path, namespace)
2090
2091         def is_drm_protected(element):
2092             return element.find(_add_ns('ContentProtection')) is not None
2093
2094         def extract_multisegment_info(element, ms_parent_info):
2095             ms_info = ms_parent_info.copy()
2096
2097             # As per [1, 5.3.9.2.2] SegmentList and SegmentTemplate share some
2098             # common attributes and elements.  We will only extract relevant
2099             # for us.
2100             def extract_common(source):
2101                 segment_timeline = source.find(_add_ns('SegmentTimeline'))
2102                 if segment_timeline is not None:
2103                     s_e = segment_timeline.findall(_add_ns('S'))
2104                     if s_e:
2105                         ms_info['total_number'] = 0
2106                         ms_info['s'] = []
2107                         for s in s_e:
2108                             r = int(s.get('r', 0))
2109                             ms_info['total_number'] += 1 + r
2110                             ms_info['s'].append({
2111                                 't': int(s.get('t', 0)),
2112                                 # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
2113                                 'd': int(s.attrib['d']),
2114                                 'r': r,
2115                             })
2116                 start_number = source.get('startNumber')
2117                 if start_number:
2118                     ms_info['start_number'] = int(start_number)
2119                 timescale = source.get('timescale')
2120                 if timescale:
2121                     ms_info['timescale'] = int(timescale)
2122                 segment_duration = source.get('duration')
2123                 if segment_duration:
2124                     ms_info['segment_duration'] = float(segment_duration)
2125
2126             def extract_Initialization(source):
2127                 initialization = source.find(_add_ns('Initialization'))
2128                 if initialization is not None:
2129                     ms_info['initialization_url'] = initialization.attrib['sourceURL']
2130
2131             segment_list = element.find(_add_ns('SegmentList'))
2132             if segment_list is not None:
2133                 extract_common(segment_list)
2134                 extract_Initialization(segment_list)
2135                 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
2136                 if segment_urls_e:
2137                     ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
2138             else:
2139                 segment_template = element.find(_add_ns('SegmentTemplate'))
2140                 if segment_template is not None:
2141                     extract_common(segment_template)
2142                     media = segment_template.get('media')
2143                     if media:
2144                         ms_info['media'] = media
2145                     initialization = segment_template.get('initialization')
2146                     if initialization:
2147                         ms_info['initialization'] = initialization
2148                     else:
2149                         extract_Initialization(segment_template)
2150             return ms_info
2151
2152         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
2153         formats = []
2154         for period in mpd_doc.findall(_add_ns('Period')):
2155             period_duration = parse_duration(period.get('duration')) or mpd_duration
2156             period_ms_info = extract_multisegment_info(period, {
2157                 'start_number': 1,
2158                 'timescale': 1,
2159             })
2160             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
2161                 if is_drm_protected(adaptation_set):
2162                     continue
2163                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
2164                 for representation in adaptation_set.findall(_add_ns('Representation')):
2165                     if is_drm_protected(representation):
2166                         continue
2167                     representation_attrib = adaptation_set.attrib.copy()
2168                     representation_attrib.update(representation.attrib)
2169                     # According to [1, 5.3.7.2, Table 9, page 41], @mimeType is mandatory
2170                     mime_type = representation_attrib['mimeType']
2171                     content_type = mime_type.split('/')[0]
2172                     if content_type == 'text':
2173                         # TODO implement WebVTT downloading
2174                         pass
2175                     elif content_type in ('video', 'audio'):
2176                         base_url = ''
2177                         for element in (representation, adaptation_set, period, mpd_doc):
2178                             base_url_e = element.find(_add_ns('BaseURL'))
2179                             if base_url_e is not None:
2180                                 base_url = base_url_e.text + base_url
2181                                 if re.match(r'^https?://', base_url):
2182                                     break
2183                         if mpd_base_url and not re.match(r'^https?://', base_url):
2184                             if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
2185                                 mpd_base_url += '/'
2186                             base_url = mpd_base_url + base_url
2187                         representation_id = representation_attrib.get('id')
2188                         lang = representation_attrib.get('lang')
2189                         url_el = representation.find(_add_ns('BaseURL'))
2190                         filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
2191                         bandwidth = int_or_none(representation_attrib.get('bandwidth'))
2192                         f = {
2193                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
2194                             'manifest_url': mpd_url,
2195                             'ext': mimetype2ext(mime_type),
2196                             'width': int_or_none(representation_attrib.get('width')),
2197                             'height': int_or_none(representation_attrib.get('height')),
2198                             'tbr': float_or_none(bandwidth, 1000),
2199                             'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
2200                             'fps': int_or_none(representation_attrib.get('frameRate')),
2201                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
2202                             'format_note': 'DASH %s' % content_type,
2203                             'filesize': filesize,
2204                             'container': mimetype2ext(mime_type) + '_dash',
2205                         }
2206                         f.update(parse_codecs(representation_attrib.get('codecs')))
2207                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
2208
2209                         def prepare_template(template_name, identifiers):
2210                             tmpl = representation_ms_info[template_name]
2211                             # First of, % characters outside $...$ templates
2212                             # must be escaped by doubling for proper processing
2213                             # by % operator string formatting used further (see
2214                             # https://github.com/ytdl-org/youtube-dl/issues/16867).
2215                             t = ''
2216                             in_template = False
2217                             for c in tmpl:
2218                                 t += c
2219                                 if c == '$':
2220                                     in_template = not in_template
2221                                 elif c == '%' and not in_template:
2222                                     t += c
2223                             # Next, $...$ templates are translated to their
2224                             # %(...) counterparts to be used with % operator
2225                             t = t.replace('$RepresentationID$', representation_id)
2226                             t = re.sub(r'\$(%s)\$' % '|'.join(identifiers), r'%(\1)d', t)
2227                             t = re.sub(r'\$(%s)%%([^$]+)\$' % '|'.join(identifiers), r'%(\1)\2', t)
2228                             t.replace('$$', '$')
2229                             return t
2230
2231                         # @initialization is a regular template like @media one
2232                         # so it should be handled just the same way (see
2233                         # https://github.com/ytdl-org/youtube-dl/issues/11605)
2234                         if 'initialization' in representation_ms_info:
2235                             initialization_template = prepare_template(
2236                                 'initialization',
2237                                 # As per [1, 5.3.9.4.2, Table 15, page 54] $Number$ and
2238                                 # $Time$ shall not be included for @initialization thus
2239                                 # only $Bandwidth$ remains
2240                                 ('Bandwidth', ))
2241                             representation_ms_info['initialization_url'] = initialization_template % {
2242                                 'Bandwidth': bandwidth,
2243                             }
2244
2245                         def location_key(location):
2246                             return 'url' if re.match(r'^https?://', location) else 'path'
2247
2248                         if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
2249
2250                             media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
2251                             media_location_key = location_key(media_template)
2252
2253                             # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
2254                             # can't be used at the same time
2255                             if '%(Number' in media_template and 's' not in representation_ms_info:
2256                                 segment_duration = None
2257                                 if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
2258                                     segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
2259                                     representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
2260                                 representation_ms_info['fragments'] = [{
2261                                     media_location_key: media_template % {
2262                                         'Number': segment_number,
2263                                         'Bandwidth': bandwidth,
2264                                     },
2265                                     'duration': segment_duration,
2266                                 } for segment_number in range(
2267                                     representation_ms_info['start_number'],
2268                                     representation_ms_info['total_number'] + representation_ms_info['start_number'])]
2269                             else:
2270                                 # $Number*$ or $Time$ in media template with S list available
2271                                 # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
2272                                 # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
2273                                 representation_ms_info['fragments'] = []
2274                                 segment_time = 0
2275                                 segment_d = None
2276                                 segment_number = representation_ms_info['start_number']
2277
2278                                 def add_segment_url():
2279                                     segment_url = media_template % {
2280                                         'Time': segment_time,
2281                                         'Bandwidth': bandwidth,
2282                                         'Number': segment_number,
2283                                     }
2284                                     representation_ms_info['fragments'].append({
2285                                         media_location_key: segment_url,
2286                                         'duration': float_or_none(segment_d, representation_ms_info['timescale']),
2287                                     })
2288
2289                                 for num, s in enumerate(representation_ms_info['s']):
2290                                     segment_time = s.get('t') or segment_time
2291                                     segment_d = s['d']
2292                                     add_segment_url()
2293                                     segment_number += 1
2294                                     for r in range(s.get('r', 0)):
2295                                         segment_time += segment_d
2296                                         add_segment_url()
2297                                         segment_number += 1
2298                                     segment_time += segment_d
2299                         elif 'segment_urls' in representation_ms_info and 's' in representation_ms_info:
2300                             # No media template
2301                             # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
2302                             # or any YouTube dashsegments video
2303                             fragments = []
2304                             segment_index = 0
2305                             timescale = representation_ms_info['timescale']
2306                             for s in representation_ms_info['s']:
2307                                 duration = float_or_none(s['d'], timescale)
2308                                 for r in range(s.get('r', 0) + 1):
2309                                     segment_uri = representation_ms_info['segment_urls'][segment_index]
2310                                     fragments.append({
2311                                         location_key(segment_uri): segment_uri,
2312                                         'duration': duration,
2313                                     })
2314                                     segment_index += 1
2315                             representation_ms_info['fragments'] = fragments
2316                         elif 'segment_urls' in representation_ms_info:
2317                             # Segment URLs with no SegmentTimeline
2318                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
2319                             # https://github.com/ytdl-org/youtube-dl/pull/14844
2320                             fragments = []
2321                             segment_duration = float_or_none(
2322                                 representation_ms_info['segment_duration'],
2323                                 representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
2324                             for segment_url in representation_ms_info['segment_urls']:
2325                                 fragment = {
2326                                     location_key(segment_url): segment_url,
2327                                 }
2328                                 if segment_duration:
2329                                     fragment['duration'] = segment_duration
2330                                 fragments.append(fragment)
2331                             representation_ms_info['fragments'] = fragments
2332                         # If there is a fragments key available then we correctly recognized fragmented media.
2333                         # Otherwise we will assume unfragmented media with direct access. Technically, such
2334                         # assumption is not necessarily correct since we may simply have no support for
2335                         # some forms of fragmented media renditions yet, but for now we'll use this fallback.
2336                         if 'fragments' in representation_ms_info:
2337                             f.update({
2338                                 # NB: mpd_url may be empty when MPD manifest is parsed from a string
2339                                 'url': mpd_url or base_url,
2340                                 'fragment_base_url': base_url,
2341                                 'fragments': [],
2342                                 'protocol': 'http_dash_segments',
2343                             })
2344                             if 'initialization_url' in representation_ms_info:
2345                                 initialization_url = representation_ms_info['initialization_url']
2346                                 if not f.get('url'):
2347                                     f['url'] = initialization_url
2348                                 f['fragments'].append({location_key(initialization_url): initialization_url})
2349                             f['fragments'].extend(representation_ms_info['fragments'])
2350                         else:
2351                             # Assuming direct URL to unfragmented media.
2352                             f['url'] = base_url
2353
2354                         # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
2355                         # is not necessarily unique within a Period thus formats with
2356                         # the same `format_id` are quite possible. There are numerous examples
2357                         # of such manifests (see https://github.com/ytdl-org/youtube-dl/issues/15111,
2358                         # https://github.com/ytdl-org/youtube-dl/issues/13919)
2359                         full_info = formats_dict.get(representation_id, {}).copy()
2360                         full_info.update(f)
2361                         formats.append(full_info)
2362                     else:
2363                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
2364         return formats
2365
2366     def _extract_ism_formats(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
2367         res = self._download_xml_handle(
2368             ism_url, video_id,
2369             note=note or 'Downloading ISM manifest',
2370             errnote=errnote or 'Failed to download ISM manifest',
2371             fatal=fatal, data=data, headers=headers, query=query)
2372         if res is False:
2373             return []
2374         ism_doc, urlh = res
2375         if ism_doc is None:
2376             return []
2377
2378         return self._parse_ism_formats(ism_doc, urlh.geturl(), ism_id)
2379
2380     def _parse_ism_formats(self, ism_doc, ism_url, ism_id=None):
2381         """
2382         Parse formats from ISM manifest.
2383         References:
2384          1. [MS-SSTR]: Smooth Streaming Protocol,
2385             https://msdn.microsoft.com/en-us/library/ff469518.aspx
2386         """
2387         if ism_doc.get('IsLive') == 'TRUE' or ism_doc.find('Protection') is not None:
2388             return []
2389
2390         duration = int(ism_doc.attrib['Duration'])
2391         timescale = int_or_none(ism_doc.get('TimeScale')) or 10000000
2392
2393         formats = []
2394         for stream in ism_doc.findall('StreamIndex'):
2395             stream_type = stream.get('Type')
2396             if stream_type not in ('video', 'audio'):
2397                 continue
2398             url_pattern = stream.attrib['Url']
2399             stream_timescale = int_or_none(stream.get('TimeScale')) or timescale
2400             stream_name = stream.get('Name')
2401             for track in stream.findall('QualityLevel'):
2402                 fourcc = track.get('FourCC', 'AACL' if track.get('AudioTag') == '255' else None)
2403                 # TODO: add support for WVC1 and WMAP
2404                 if fourcc not in ('H264', 'AVC1', 'AACL'):
2405                     self.report_warning('%s is not a supported codec' % fourcc)
2406                     continue
2407                 tbr = int(track.attrib['Bitrate']) // 1000
2408                 # [1] does not mention Width and Height attributes. However,
2409                 # they're often present while MaxWidth and MaxHeight are
2410                 # missing, so should be used as fallbacks
2411                 width = int_or_none(track.get('MaxWidth') or track.get('Width'))
2412                 height = int_or_none(track.get('MaxHeight') or track.get('Height'))
2413                 sampling_rate = int_or_none(track.get('SamplingRate'))
2414
2415                 track_url_pattern = re.sub(r'{[Bb]itrate}', track.attrib['Bitrate'], url_pattern)
2416                 track_url_pattern = compat_urlparse.urljoin(ism_url, track_url_pattern)
2417
2418                 fragments = []
2419                 fragment_ctx = {
2420                     'time': 0,
2421                 }
2422                 stream_fragments = stream.findall('c')
2423                 for stream_fragment_index, stream_fragment in enumerate(stream_fragments):
2424                     fragment_ctx['time'] = int_or_none(stream_fragment.get('t')) or fragment_ctx['time']
2425                     fragment_repeat = int_or_none(stream_fragment.get('r')) or 1
2426                     fragment_ctx['duration'] = int_or_none(stream_fragment.get('d'))
2427                     if not fragment_ctx['duration']:
2428                         try:
2429                             next_fragment_time = int(stream_fragment[stream_fragment_index + 1].attrib['t'])
2430                         except IndexError:
2431                             next_fragment_time = duration
2432                         fragment_ctx['duration'] = (next_fragment_time - fragment_ctx['time']) / fragment_repeat
2433                     for _ in range(fragment_repeat):
2434                         fragments.append({
2435                             'url': re.sub(r'{start[ _]time}', compat_str(fragment_ctx['time']), track_url_pattern),
2436                             'duration': fragment_ctx['duration'] / stream_timescale,
2437                         })
2438                         fragment_ctx['time'] += fragment_ctx['duration']
2439
2440                 format_id = []
2441                 if ism_id:
2442                     format_id.append(ism_id)
2443                 if stream_name:
2444                     format_id.append(stream_name)
2445                 format_id.append(compat_str(tbr))
2446
2447                 formats.append({
2448                     'format_id': '-'.join(format_id),
2449                     'url': ism_url,
2450                     'manifest_url': ism_url,
2451                     'ext': 'ismv' if stream_type == 'video' else 'isma',
2452                     'width': width,
2453                     'height': height,
2454                     'tbr': tbr,
2455                     'asr': sampling_rate,
2456                     'vcodec': 'none' if stream_type == 'audio' else fourcc,
2457                     'acodec': 'none' if stream_type == 'video' else fourcc,
2458                     'protocol': 'ism',
2459                     'fragments': fragments,
2460                     '_download_params': {
2461                         'duration': duration,
2462                         'timescale': stream_timescale,
2463                         'width': width or 0,
2464                         'height': height or 0,
2465                         'fourcc': fourcc,
2466                         'codec_private_data': track.get('CodecPrivateData'),
2467                         'sampling_rate': sampling_rate,
2468                         'channels': int_or_none(track.get('Channels', 2)),
2469                         'bits_per_sample': int_or_none(track.get('BitsPerSample', 16)),
2470                         'nal_unit_length_field': int_or_none(track.get('NALUnitLengthField', 4)),
2471                     },
2472                 })
2473         return formats
2474
2475     def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8', mpd_id=None, preference=None):
2476         def absolute_url(item_url):
2477             return urljoin(base_url, item_url)
2478
2479         def parse_content_type(content_type):
2480             if not content_type:
2481                 return {}
2482             ctr = re.search(r'(?P<mimetype>[^/]+/[^;]+)(?:;\s*codecs="?(?P<codecs>[^"]+))?', content_type)
2483             if ctr:
2484                 mimetype, codecs = ctr.groups()
2485                 f = parse_codecs(codecs)
2486                 f['ext'] = mimetype2ext(mimetype)
2487                 return f
2488             return {}
2489
2490         def _media_formats(src, cur_media_type, type_info={}):
2491             full_url = absolute_url(src)
2492             ext = type_info.get('ext') or determine_ext(full_url)
2493             if ext == 'm3u8':
2494                 is_plain_url = False
2495                 formats = self._extract_m3u8_formats(
2496                     full_url, video_id, ext='mp4',
2497                     entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id,
2498                     preference=preference, fatal=False)
2499             elif ext == 'mpd':
2500                 is_plain_url = False
2501                 formats = self._extract_mpd_formats(
2502                     full_url, video_id, mpd_id=mpd_id, fatal=False)
2503             else:
2504                 is_plain_url = True
2505                 formats = [{
2506                     'url': full_url,
2507                     'vcodec': 'none' if cur_media_type == 'audio' else None,
2508                 }]
2509             return is_plain_url, formats
2510
2511         entries = []
2512         # amp-video and amp-audio are very similar to their HTML5 counterparts
2513         # so we wll include them right here (see
2514         # https://www.ampproject.org/docs/reference/components/amp-video)
2515         media_tags = [(media_tag, media_type, '')
2516                       for media_tag, media_type
2517                       in re.findall(r'(?s)(<(?:amp-)?(video|audio)[^>]*/>)', webpage)]
2518         media_tags.extend(re.findall(
2519             # We only allow video|audio followed by a whitespace or '>'.
2520             # Allowing more characters may end up in significant slow down (see
2521             # https://github.com/ytdl-org/youtube-dl/issues/11979, example URL:
2522             # http://www.porntrex.com/maps/videositemap.xml).
2523             r'(?s)(<(?P<tag>(?:amp-)?(?:video|audio))(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
2524         for media_tag, media_type, media_content in media_tags:
2525             media_info = {
2526                 'formats': [],
2527                 'subtitles': {},
2528             }
2529             media_attributes = extract_attributes(media_tag)
2530             src = strip_or_none(media_attributes.get('src'))
2531             if src:
2532                 _, formats = _media_formats(src, media_type)
2533                 media_info['formats'].extend(formats)
2534             media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
2535             if media_content:
2536                 for source_tag in re.findall(r'<source[^>]+>', media_content):
2537                     s_attr = extract_attributes(source_tag)
2538                     # data-video-src and data-src are non standard but seen
2539                     # several times in the wild
2540                     src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src')))
2541                     if not src:
2542                         continue
2543                     f = parse_content_type(s_attr.get('type'))
2544                     is_plain_url, formats = _media_formats(src, media_type, f)
2545                     if is_plain_url:
2546                         # width, height, res, label and title attributes are
2547                         # all not standard but seen several times in the wild
2548                         labels = [
2549                             s_attr.get(lbl)
2550                             for lbl in ('label', 'title')
2551                             if str_or_none(s_attr.get(lbl))
2552                         ]
2553                         width = int_or_none(s_attr.get('width'))
2554                         height = (int_or_none(s_attr.get('height'))
2555                                   or int_or_none(s_attr.get('res')))
2556                         if not width or not height:
2557                             for lbl in labels:
2558                                 resolution = parse_resolution(lbl)
2559                                 if not resolution:
2560                                     continue
2561                                 width = width or resolution.get('width')
2562                                 height = height or resolution.get('height')
2563                         for lbl in labels:
2564                             tbr = parse_bitrate(lbl)
2565                             if tbr:
2566                                 break
2567                         else:
2568                             tbr = None
2569                         f.update({
2570                             'width': width,
2571                             'height': height,
2572                             'tbr': tbr,
2573                             'format_id': s_attr.get('label') or s_attr.get('title'),
2574                         })
2575                         f.update(formats[0])
2576                         media_info['formats'].append(f)
2577                     else:
2578                         media_info['formats'].extend(formats)
2579                 for track_tag in re.findall(r'<track[^>]+>', media_content):
2580                     track_attributes = extract_attributes(track_tag)
2581                     kind = track_attributes.get('kind')
2582                     if not kind or kind in ('subtitles', 'captions'):
2583                         src = strip_or_none(track_attributes.get('src'))
2584                         if not src:
2585                             continue
2586                         lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')
2587                         media_info['subtitles'].setdefault(lang, []).append({
2588                             'url': absolute_url(src),
2589                         })
2590             for f in media_info['formats']:
2591                 f.setdefault('http_headers', {})['Referer'] = base_url
2592             if media_info['formats'] or media_info['subtitles']:
2593                 entries.append(media_info)
2594         return entries
2595
2596     def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
2597         formats = []
2598         hdcore_sign = 'hdcore=3.7.0'
2599         f4m_url = re.sub(r'(https?://[^/]+)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
2600         hds_host = hosts.get('hds')
2601         if hds_host:
2602             f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)
2603         if 'hdcore=' not in f4m_url:
2604             f4m_url += ('&' if '?' in f4m_url else '?') + hdcore_sign
2605         f4m_formats = self._extract_f4m_formats(
2606             f4m_url, video_id, f4m_id='hds', fatal=False)
2607         for entry in f4m_formats:
2608             entry.update({'extra_param_to_segment_url': hdcore_sign})
2609         formats.extend(f4m_formats)
2610         m3u8_url = re.sub(r'(https?://[^/]+)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
2611         hls_host = hosts.get('hls')
2612         if hls_host:
2613             m3u8_url = re.sub(r'(https?://)[^/]+', r'\1' + hls_host, m3u8_url)
2614         formats.extend(self._extract_m3u8_formats(
2615             m3u8_url, video_id, 'mp4', 'm3u8_native',
2616             m3u8_id='hls', fatal=False))
2617         return formats
2618
2619     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
2620         query = compat_urlparse.urlparse(url).query
2621         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
2622         mobj = re.search(
2623             r'(?:(?:http|rtmp|rtsp)(?P<s>s)?:)?(?P<url>//[^?]+)', url)
2624         url_base = mobj.group('url')
2625         http_base_url = '%s%s:%s' % ('http', mobj.group('s') or '', url_base)
2626         formats = []
2627
2628         def manifest_url(manifest):
2629             m_url = '%s/%s' % (http_base_url, manifest)
2630             if query:
2631                 m_url += '?%s' % query
2632             return m_url
2633
2634         if 'm3u8' not in skip_protocols:
2635             formats.extend(self._extract_m3u8_formats(
2636                 manifest_url('playlist.m3u8'), video_id, 'mp4',
2637                 m3u8_entry_protocol, m3u8_id='hls', fatal=False))
2638         if 'f4m' not in skip_protocols:
2639             formats.extend(self._extract_f4m_formats(
2640                 manifest_url('manifest.f4m'),
2641                 video_id, f4m_id='hds', fatal=False))
2642         if 'dash' not in skip_protocols:
2643             formats.extend(self._extract_mpd_formats(
2644                 manifest_url('manifest.mpd'),
2645                 video_id, mpd_id='dash', fatal=False))
2646         if re.search(r'(?:/smil:|\.smil)', url_base):
2647             if 'smil' not in skip_protocols:
2648                 rtmp_formats = self._extract_smil_formats(
2649                     manifest_url('jwplayer.smil'),
2650                     video_id, fatal=False)
2651                 for rtmp_format in rtmp_formats:
2652                     rtsp_format = rtmp_format.copy()
2653                     rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
2654                     del rtsp_format['play_path']
2655                     del rtsp_format['ext']
2656                     rtsp_format.update({
2657                         'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
2658                         'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
2659                         'protocol': 'rtsp',
2660                     })
2661                     formats.extend([rtmp_format, rtsp_format])
2662         else:
2663             for protocol in ('rtmp', 'rtsp'):
2664                 if protocol not in skip_protocols:
2665                     formats.append({
2666                         'url': '%s:%s' % (protocol, url_base),
2667                         'format_id': protocol,
2668                         'protocol': protocol,
2669                     })
2670         return formats
2671
2672     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
2673         mobj = re.search(
2674             r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
2675             webpage)
2676         if mobj:
2677             try:
2678                 jwplayer_data = self._parse_json(mobj.group('options'),
2679                                                  video_id=video_id,
2680                                                  transform_source=transform_source)
2681             except ExtractorError:
2682                 pass
2683             else:
2684                 if isinstance(jwplayer_data, dict):
2685                     return jwplayer_data
2686
2687     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
2688         jwplayer_data = self._find_jwplayer_data(
2689             webpage, video_id, transform_source=js_to_json)
2690         return self._parse_jwplayer_data(
2691             jwplayer_data, video_id, *args, **kwargs)
2692
2693     def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
2694                              m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2695         # JWPlayer backward compatibility: flattened playlists
2696         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
2697         if 'playlist' not in jwplayer_data:
2698             jwplayer_data = {'playlist': [jwplayer_data]}
2699
2700         entries = []
2701
2702         # JWPlayer backward compatibility: single playlist item
2703         # https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
2704         if not isinstance(jwplayer_data['playlist'], list):
2705             jwplayer_data['playlist'] = [jwplayer_data['playlist']]
2706
2707         for video_data in jwplayer_data['playlist']:
2708             # JWPlayer backward compatibility: flattened sources
2709             # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
2710             if 'sources' not in video_data:
2711                 video_data['sources'] = [video_data]
2712
2713             this_video_id = video_id or video_data['mediaid']
2714
2715             formats = self._parse_jwplayer_formats(
2716                 video_data['sources'], video_id=this_video_id, m3u8_id=m3u8_id,
2717                 mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
2718
2719             subtitles = {}
2720             tracks = video_data.get('tracks')
2721             if tracks and isinstance(tracks, list):
2722                 for track in tracks:
2723                     if not isinstance(track, dict):
2724                         continue
2725                     track_kind = track.get('kind')
2726                     if not track_kind or not isinstance(track_kind, compat_str):
2727                         continue
2728                     if track_kind.lower() not in ('captions', 'subtitles'):
2729                         continue
2730                     track_url = urljoin(base_url, track.get('file'))
2731                     if not track_url:
2732                         continue
2733                     subtitles.setdefault(track.get('label') or 'en', []).append({
2734                         'url': self._proto_relative_url(track_url)
2735                     })
2736
2737             entry = {
2738                 'id': this_video_id,
2739                 'title': unescapeHTML(video_data['title'] if require_title else video_data.get('title')),
2740                 'description': clean_html(video_data.get('description')),
2741                 'thumbnail': urljoin(base_url, self._proto_relative_url(video_data.get('image'))),
2742                 'timestamp': int_or_none(video_data.get('pubdate')),
2743                 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
2744                 'subtitles': subtitles,
2745             }
2746             # https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
2747             if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
2748                 entry.update({
2749                     '_type': 'url_transparent',
2750                     'url': formats[0]['url'],
2751                 })
2752             else:
2753                 self._sort_formats(formats)
2754                 entry['formats'] = formats
2755             entries.append(entry)
2756         if len(entries) == 1:
2757             return entries[0]
2758         else:
2759             return self.playlist_result(entries)
2760
2761     def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
2762                                 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2763         urls = []
2764         formats = []
2765         for source in jwplayer_sources_data:
2766             if not isinstance(source, dict):
2767                 continue
2768             source_url = urljoin(
2769                 base_url, self._proto_relative_url(source.get('file')))
2770             if not source_url or source_url in urls:
2771                 continue
2772             urls.append(source_url)
2773             source_type = source.get('type') or ''
2774             ext = mimetype2ext(source_type) or determine_ext(source_url)
2775             if source_type == 'hls' or ext == 'm3u8':
2776                 formats.extend(self._extract_m3u8_formats(
2777                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
2778                     m3u8_id=m3u8_id, fatal=False))
2779             elif source_type == 'dash' or ext == 'mpd':
2780                 formats.extend(self._extract_mpd_formats(
2781                     source_url, video_id, mpd_id=mpd_id, fatal=False))
2782             elif ext == 'smil':
2783                 formats.extend(self._extract_smil_formats(
2784                     source_url, video_id, fatal=False))
2785             # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
2786             elif source_type.startswith('audio') or ext in (
2787                     'oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
2788                 formats.append({
2789                     'url': source_url,
2790                     'vcodec': 'none',
2791                     'ext': ext,
2792                 })
2793             else:
2794                 height = int_or_none(source.get('height'))
2795                 if height is None:
2796                     # Often no height is provided but there is a label in
2797                     # format like "1080p", "720p SD", or 1080.
2798                     height = int_or_none(self._search_regex(
2799                         r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
2800                         'height', default=None))
2801                 a_format = {
2802                     'url': source_url,
2803                     'width': int_or_none(source.get('width')),
2804                     'height': height,
2805                     'tbr': int_or_none(source.get('bitrate')),
2806                     'ext': ext,
2807                 }
2808                 if source_url.startswith('rtmp'):
2809                     a_format['ext'] = 'flv'
2810                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
2811                     # of jwplayer.flash.swf
2812                     rtmp_url_parts = re.split(
2813                         r'((?:mp4|mp3|flv):)', source_url, 1)
2814                     if len(rtmp_url_parts) == 3:
2815                         rtmp_url, prefix, play_path = rtmp_url_parts
2816                         a_format.update({
2817                             'url': rtmp_url,
2818                             'play_path': prefix + play_path,
2819                         })
2820                     if rtmp_params:
2821                         a_format.update(rtmp_params)
2822                 formats.append(a_format)
2823         return formats
2824
2825     def _live_title(self, name):
2826         """ Generate the title for a live video """
2827         now = datetime.datetime.now()
2828         now_str = now.strftime('%Y-%m-%d %H:%M')
2829         return name + ' ' + now_str
2830
2831     def _int(self, v, name, fatal=False, **kwargs):
2832         res = int_or_none(v, **kwargs)
2833         if 'get_attr' in kwargs:
2834             print(getattr(v, kwargs['get_attr']))
2835         if res is None:
2836             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2837             if fatal:
2838                 raise ExtractorError(msg)
2839             else:
2840                 self._downloader.report_warning(msg)
2841         return res
2842
2843     def _float(self, v, name, fatal=False, **kwargs):
2844         res = float_or_none(v, **kwargs)
2845         if res is None:
2846             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2847             if fatal:
2848                 raise ExtractorError(msg)
2849             else:
2850                 self._downloader.report_warning(msg)
2851         return res
2852
2853     def _set_cookie(self, domain, name, value, expire_time=None, port=None,
2854                     path='/', secure=False, discard=False, rest={}, **kwargs):
2855         cookie = compat_cookiejar_Cookie(
2856             0, name, value, port, port is not None, domain, True,
2857             domain.startswith('.'), path, True, secure, expire_time,
2858             discard, None, None, rest)
2859         self._downloader.cookiejar.set_cookie(cookie)
2860
2861     def _get_cookies(self, url):
2862         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
2863         req = sanitized_Request(url)
2864         self._downloader.cookiejar.add_cookie_header(req)
2865         return compat_cookies.SimpleCookie(req.get_header('Cookie'))
2866
2867     def _apply_first_set_cookie_header(self, url_handle, cookie):
2868         """
2869         Apply first Set-Cookie header instead of the last. Experimental.
2870
2871         Some sites (e.g. [1-3]) may serve two cookies under the same name
2872         in Set-Cookie header and expect the first (old) one to be set rather
2873         than second (new). However, as of RFC6265 the newer one cookie
2874         should be set into cookie store what actually happens.
2875         We will workaround this issue by resetting the cookie to
2876         the first one manually.
2877         1. https://new.vk.com/
2878         2. https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201
2879         3. https://learning.oreilly.com/
2880         """
2881         for header, cookies in url_handle.headers.items():
2882             if header.lower() != 'set-cookie':
2883                 continue
2884             if sys.version_info[0] >= 3:
2885                 cookies = cookies.encode('iso-8859-1')
2886             cookies = cookies.decode('utf-8')
2887             cookie_value = re.search(
2888                 r'%s=(.+?);.*?\b[Dd]omain=(.+?)(?:[,;]|$)' % cookie, cookies)
2889             if cookie_value:
2890                 value, domain = cookie_value.groups()
2891                 self._set_cookie(domain, cookie, value)
2892                 break
2893
2894     def get_testcases(self, include_onlymatching=False):
2895         t = getattr(self, '_TEST', None)
2896         if t:
2897             assert not hasattr(self, '_TESTS'), \
2898                 '%s has _TEST and _TESTS' % type(self).__name__
2899             tests = [t]
2900         else:
2901             tests = getattr(self, '_TESTS', [])
2902         for t in tests:
2903             if not include_onlymatching and t.get('only_matching', False):
2904                 continue
2905             t['name'] = type(self).__name__[:-len('IE')]
2906             yield t
2907
2908     def is_suitable(self, age_limit):
2909         """ Test whether the extractor is generally suitable for the given
2910         age limit (i.e. pornographic sites are not, all others usually are) """
2911
2912         any_restricted = False
2913         for tc in self.get_testcases(include_onlymatching=False):
2914             if tc.get('playlist', []):
2915                 tc = tc['playlist'][0]
2916             is_restricted = age_restricted(
2917                 tc.get('info_dict', {}).get('age_limit'), age_limit)
2918             if not is_restricted:
2919                 return True
2920             any_restricted = any_restricted or is_restricted
2921         return not any_restricted
2922
2923     def extract_subtitles(self, *args, **kwargs):
2924         if (self._downloader.params.get('writesubtitles', False)
2925                 or self._downloader.params.get('listsubtitles')):
2926             return self._get_subtitles(*args, **kwargs)
2927         return {}
2928
2929     def _get_subtitles(self, *args, **kwargs):
2930         raise NotImplementedError('This method must be implemented by subclasses')
2931
2932     @staticmethod
2933     def _merge_subtitle_items(subtitle_list1, subtitle_list2):
2934         """ Merge subtitle items for one language. Items with duplicated URLs
2935         will be dropped. """
2936         list1_urls = set([item['url'] for item in subtitle_list1])
2937         ret = list(subtitle_list1)
2938         ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
2939         return ret
2940
2941     @classmethod
2942     def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
2943         """ Merge two subtitle dictionaries, language by language. """
2944         ret = dict(subtitle_dict1)
2945         for lang in subtitle_dict2:
2946             ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
2947         return ret
2948
2949     def extract_automatic_captions(self, *args, **kwargs):
2950         if (self._downloader.params.get('writeautomaticsub', False)
2951                 or self._downloader.params.get('listsubtitles')):
2952             return self._get_automatic_captions(*args, **kwargs)
2953         return {}
2954
2955     def _get_automatic_captions(self, *args, **kwargs):
2956         raise NotImplementedError('This method must be implemented by subclasses')
2957
2958     def mark_watched(self, *args, **kwargs):
2959         if (self._downloader.params.get('mark_watched', False)
2960                 and (self._get_login_info()[0] is not None
2961                      or self._downloader.params.get('cookiefile') is not None)):
2962             self._mark_watched(*args, **kwargs)
2963
2964     def _mark_watched(self, *args, **kwargs):
2965         raise NotImplementedError('This method must be implemented by subclasses')
2966
2967     def geo_verification_headers(self):
2968         headers = {}
2969         geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
2970         if geo_verification_proxy:
2971             headers['Ytdl-request-proxy'] = geo_verification_proxy
2972         return headers
2973
2974     def _generic_id(self, url):
2975         return compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
2976
2977     def _generic_title(self, url):
2978         return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
2979
2980
2981 class SearchInfoExtractor(InfoExtractor):
2982     """
2983     Base class for paged search queries extractors.
2984     They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
2985     Instances should define _SEARCH_KEY and _MAX_RESULTS.
2986     """
2987
2988     @classmethod
2989     def _make_valid_url(cls):
2990         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
2991
2992     @classmethod
2993     def suitable(cls, url):
2994         return re.match(cls._make_valid_url(), url) is not None
2995
2996     def _real_extract(self, query):
2997         mobj = re.match(self._make_valid_url(), query)
2998         if mobj is None:
2999             raise ExtractorError('Invalid search query "%s"' % query)
3000
3001         prefix = mobj.group('prefix')
3002         query = mobj.group('query')
3003         if prefix == '':
3004             return self._get_n_results(query, 1)
3005         elif prefix == 'all':
3006             return self._get_n_results(query, self._MAX_RESULTS)
3007         else:
3008             n = int(prefix)
3009             if n <= 0:
3010                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
3011             elif n > self._MAX_RESULTS:
3012                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
3013                 n = self._MAX_RESULTS
3014             return self._get_n_results(query, n)
3015
3016     def _get_n_results(self, query, n):
3017         """Get a specified number of results for a query"""
3018         raise NotImplementedError('This method must be implemented by subclasses')
3019
3020     @property
3021     def SEARCH_KEY(self):
3022         return self._SEARCH_KEY