]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/common.py
[youporn] Add support form generic embeds
[youtube-dl] / youtube_dl / extractor / common.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import base64
5 import datetime
6 import hashlib
7 import json
8 import netrc
9 import os
10 import random
11 import re
12 import socket
13 import sys
14 import time
15 import math
16
17 from ..compat import (
18     compat_cookiejar,
19     compat_cookies,
20     compat_etree_Element,
21     compat_etree_fromstring,
22     compat_getpass,
23     compat_integer_types,
24     compat_http_client,
25     compat_os_name,
26     compat_str,
27     compat_urllib_error,
28     compat_urllib_parse_unquote,
29     compat_urllib_parse_urlencode,
30     compat_urllib_request,
31     compat_urlparse,
32     compat_xml_parse_error,
33 )
34 from ..downloader.f4m import (
35     get_base_url,
36     remove_encrypted_media,
37 )
38 from ..utils import (
39     NO_DEFAULT,
40     age_restricted,
41     base_url,
42     bug_reports_message,
43     clean_html,
44     compiled_regex_type,
45     determine_ext,
46     determine_protocol,
47     dict_get,
48     error_to_compat_str,
49     ExtractorError,
50     extract_attributes,
51     fix_xml_ampersands,
52     float_or_none,
53     GeoRestrictedError,
54     GeoUtils,
55     int_or_none,
56     js_to_json,
57     JSON_LD_RE,
58     mimetype2ext,
59     orderedSet,
60     parse_bitrate,
61     parse_codecs,
62     parse_duration,
63     parse_iso8601,
64     parse_m3u8_attributes,
65     parse_resolution,
66     RegexNotFoundError,
67     sanitized_Request,
68     sanitize_filename,
69     str_or_none,
70     strip_or_none,
71     unescapeHTML,
72     unified_strdate,
73     unified_timestamp,
74     update_Request,
75     update_url_query,
76     urljoin,
77     url_basename,
78     url_or_none,
79     xpath_element,
80     xpath_text,
81     xpath_with_ns,
82 )
83
84
85 class InfoExtractor(object):
86     """Information Extractor class.
87
88     Information extractors are the classes that, given a URL, extract
89     information about the video (or videos) the URL refers to. This
90     information includes the real video URL, the video title, author and
91     others. The information is stored in a dictionary which is then
92     passed to the YoutubeDL. The YoutubeDL processes this
93     information possibly downloading the video to the file system, among
94     other possible outcomes.
95
96     The type field determines the type of the result.
97     By far the most common value (and the default if _type is missing) is
98     "video", which indicates a single video.
99
100     For a video, the dictionaries must include the following fields:
101
102     id:             Video identifier.
103     title:          Video title, unescaped.
104
105     Additionally, it must contain either a formats entry or a url one:
106
107     formats:        A list of dictionaries for each format available, ordered
108                     from worst to best quality.
109
110                     Potential fields:
111                     * url        The mandatory URL representing the media:
112                                    for plain file media - HTTP URL of this file,
113                                    for RTMP - RTMP URL,
114                                    for HLS - URL of the M3U8 media playlist,
115                                    for HDS - URL of the F4M manifest,
116                                    for DASH
117                                      - HTTP URL to plain file media (in case of
118                                        unfragmented media)
119                                      - URL of the MPD manifest or base URL
120                                        representing the media if MPD manifest
121                                        is parsed from a string (in case of
122                                        fragmented media)
123                                    for MSS - URL of the ISM manifest.
124                     * manifest_url
125                                  The URL of the manifest file in case of
126                                  fragmented media:
127                                    for HLS - URL of the M3U8 master playlist,
128                                    for HDS - URL of the F4M manifest,
129                                    for DASH - URL of the MPD manifest,
130                                    for MSS - URL of the ISM manifest.
131                     * ext        Will be calculated from URL if missing
132                     * format     A human-readable description of the format
133                                  ("mp4 container with h264/opus").
134                                  Calculated from the format_id, width, height.
135                                  and format_note fields if missing.
136                     * format_id  A short description of the format
137                                  ("mp4_h264_opus" or "19").
138                                 Technically optional, but strongly recommended.
139                     * format_note Additional info about the format
140                                  ("3D" or "DASH video")
141                     * width      Width of the video, if known
142                     * height     Height of the video, if known
143                     * resolution Textual description of width and height
144                     * tbr        Average bitrate of audio and video in KBit/s
145                     * abr        Average audio bitrate in KBit/s
146                     * acodec     Name of the audio codec in use
147                     * asr        Audio sampling rate in Hertz
148                     * vbr        Average video bitrate in KBit/s
149                     * fps        Frame rate
150                     * vcodec     Name of the video codec in use
151                     * container  Name of the container format
152                     * filesize   The number of bytes, if known in advance
153                     * filesize_approx  An estimate for the number of bytes
154                     * player_url SWF Player URL (used for rtmpdump).
155                     * protocol   The protocol that will be used for the actual
156                                  download, lower-case.
157                                  "http", "https", "rtsp", "rtmp", "rtmpe",
158                                  "m3u8", "m3u8_native" or "http_dash_segments".
159                     * fragment_base_url
160                                  Base URL for fragments. Each fragment's path
161                                  value (if present) will be relative to
162                                  this URL.
163                     * fragments  A list of fragments of a fragmented media.
164                                  Each fragment entry must contain either an url
165                                  or a path. If an url is present it should be
166                                  considered by a client. Otherwise both path and
167                                  fragment_base_url must be present. Here is
168                                  the list of all potential fields:
169                                  * "url" - fragment's URL
170                                  * "path" - fragment's path relative to
171                                             fragment_base_url
172                                  * "duration" (optional, int or float)
173                                  * "filesize" (optional, int)
174                     * preference Order number of this format. If this field is
175                                  present and not None, the formats get sorted
176                                  by this field, regardless of all other values.
177                                  -1 for default (order by other properties),
178                                  -2 or smaller for less than default.
179                                  < -1000 to hide the format (if there is
180                                     another one which is strictly better)
181                     * language   Language code, e.g. "de" or "en-US".
182                     * language_preference  Is this in the language mentioned in
183                                  the URL?
184                                  10 if it's what the URL is about,
185                                  -1 for default (don't know),
186                                  -10 otherwise, other values reserved for now.
187                     * quality    Order number of the video quality of this
188                                  format, irrespective of the file format.
189                                  -1 for default (order by other properties),
190                                  -2 or smaller for less than default.
191                     * source_preference  Order number for this video source
192                                   (quality takes higher priority)
193                                  -1 for default (order by other properties),
194                                  -2 or smaller for less than default.
195                     * http_headers  A dictionary of additional HTTP headers
196                                  to add to the request.
197                     * stretched_ratio  If given and not 1, indicates that the
198                                  video's pixels are not square.
199                                  width : height ratio as float.
200                     * no_resume  The server does not support resuming the
201                                  (HTTP or RTMP) download. Boolean.
202                     * downloader_options  A dictionary of downloader options as
203                                  described in FileDownloader
204
205     url:            Final video URL.
206     ext:            Video filename extension.
207     format:         The video format, defaults to ext (used for --get-format)
208     player_url:     SWF Player URL (used for rtmpdump).
209
210     The following fields are optional:
211
212     alt_title:      A secondary title of the video.
213     display_id      An alternative identifier for the video, not necessarily
214                     unique, but available before title. Typically, id is
215                     something like "4234987", title "Dancing naked mole rats",
216                     and display_id "dancing-naked-mole-rats"
217     thumbnails:     A list of dictionaries, with the following entries:
218                         * "id" (optional, string) - Thumbnail format ID
219                         * "url"
220                         * "preference" (optional, int) - quality of the image
221                         * "width" (optional, int)
222                         * "height" (optional, int)
223                         * "resolution" (optional, string "{width}x{height}",
224                                         deprecated)
225                         * "filesize" (optional, int)
226     thumbnail:      Full URL to a video thumbnail image.
227     description:    Full video description.
228     uploader:       Full name of the video uploader.
229     license:        License name the video is licensed under.
230     creator:        The creator of the video.
231     release_date:   The date (YYYYMMDD) when the video was released.
232     timestamp:      UNIX timestamp of the moment the video became available.
233     upload_date:    Video upload date (YYYYMMDD).
234                     If not explicitly set, calculated from timestamp.
235     uploader_id:    Nickname or id of the video uploader.
236     uploader_url:   Full URL to a personal webpage of the video uploader.
237     channel:        Full name of the channel the video is uploaded on.
238                     Note that channel fields may or may not repeat uploader
239                     fields. This depends on a particular extractor.
240     channel_id:     Id of the channel.
241     channel_url:    Full URL to a channel webpage.
242     location:       Physical location where the video was filmed.
243     subtitles:      The available subtitles as a dictionary in the format
244                     {tag: subformats}. "tag" is usually a language code, and
245                     "subformats" is a list sorted from lower to higher
246                     preference, each element is a dictionary with the "ext"
247                     entry and one of:
248                         * "data": The subtitles file contents
249                         * "url": A URL pointing to the subtitles file
250                     "ext" will be calculated from URL if missing
251     automatic_captions: Like 'subtitles', used by the YoutubeIE for
252                     automatically generated captions
253     duration:       Length of the video in seconds, as an integer or float.
254     view_count:     How many users have watched the video on the platform.
255     like_count:     Number of positive ratings of the video
256     dislike_count:  Number of negative ratings of the video
257     repost_count:   Number of reposts of the video
258     average_rating: Average rating give by users, the scale used depends on the webpage
259     comment_count:  Number of comments on the video
260     comments:       A list of comments, each with one or more of the following
261                     properties (all but one of text or html optional):
262                         * "author" - human-readable name of the comment author
263                         * "author_id" - user ID of the comment author
264                         * "id" - Comment ID
265                         * "html" - Comment as HTML
266                         * "text" - Plain text of the comment
267                         * "timestamp" - UNIX timestamp of comment
268                         * "parent" - ID of the comment this one is replying to.
269                                      Set to "root" to indicate that this is a
270                                      comment to the original video.
271     age_limit:      Age restriction for the video, as an integer (years)
272     webpage_url:    The URL to the video webpage, if given to youtube-dl it
273                     should allow to get the same result again. (It will be set
274                     by YoutubeDL if it's missing)
275     categories:     A list of categories that the video falls in, for example
276                     ["Sports", "Berlin"]
277     tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
278     is_live:        True, False, or None (=unknown). Whether this video is a
279                     live stream that goes on instead of a fixed-length video.
280     start_time:     Time in seconds where the reproduction should start, as
281                     specified in the URL.
282     end_time:       Time in seconds where the reproduction should end, as
283                     specified in the URL.
284     chapters:       A list of dictionaries, with the following entries:
285                         * "start_time" - The start time of the chapter in seconds
286                         * "end_time" - The end time of the chapter in seconds
287                         * "title" (optional, string)
288
289     The following fields should only be used when the video belongs to some logical
290     chapter or section:
291
292     chapter:        Name or title of the chapter the video belongs to.
293     chapter_number: Number of the chapter the video belongs to, as an integer.
294     chapter_id:     Id of the chapter the video belongs to, as a unicode string.
295
296     The following fields should only be used when the video is an episode of some
297     series, programme or podcast:
298
299     series:         Title of the series or programme the video episode belongs to.
300     season:         Title of the season the video episode belongs to.
301     season_number:  Number of the season the video episode belongs to, as an integer.
302     season_id:      Id of the season the video episode belongs to, as a unicode string.
303     episode:        Title of the video episode. Unlike mandatory video title field,
304                     this field should denote the exact title of the video episode
305                     without any kind of decoration.
306     episode_number: Number of the video episode within a season, as an integer.
307     episode_id:     Id of the video episode, as a unicode string.
308
309     The following fields should only be used when the media is a track or a part of
310     a music album:
311
312     track:          Title of the track.
313     track_number:   Number of the track within an album or a disc, as an integer.
314     track_id:       Id of the track (useful in case of custom indexing, e.g. 6.iii),
315                     as a unicode string.
316     artist:         Artist(s) of the track.
317     genre:          Genre(s) of the track.
318     album:          Title of the album the track belongs to.
319     album_type:     Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc).
320     album_artist:   List of all artists appeared on the album (e.g.
321                     "Ash Borer / Fell Voices" or "Various Artists", useful for splits
322                     and compilations).
323     disc_number:    Number of the disc or other physical medium the track belongs to,
324                     as an integer.
325     release_year:   Year (YYYY) when the album was released.
326
327     Unless mentioned otherwise, the fields should be Unicode strings.
328
329     Unless mentioned otherwise, None is equivalent to absence of information.
330
331
332     _type "playlist" indicates multiple videos.
333     There must be a key "entries", which is a list, an iterable, or a PagedList
334     object, each element of which is a valid dictionary by this specification.
335
336     Additionally, playlists can have "id", "title", "description", "uploader",
337     "uploader_id", "uploader_url" attributes with the same semantics as videos
338     (see above).
339
340
341     _type "multi_video" indicates that there are multiple videos that
342     form a single show, for examples multiple acts of an opera or TV episode.
343     It must have an entries key like a playlist and contain all the keys
344     required for a video at the same time.
345
346
347     _type "url" indicates that the video must be extracted from another
348     location, possibly by a different extractor. Its only required key is:
349     "url" - the next URL to extract.
350     The key "ie_key" can be set to the class name (minus the trailing "IE",
351     e.g. "Youtube") if the extractor class is known in advance.
352     Additionally, the dictionary may have any properties of the resolved entity
353     known in advance, for example "title" if the title of the referred video is
354     known ahead of time.
355
356
357     _type "url_transparent" entities have the same specification as "url", but
358     indicate that the given additional information is more precise than the one
359     associated with the resolved URL.
360     This is useful when a site employs a video service that hosts the video and
361     its technical metadata, but that video service does not embed a useful
362     title, description etc.
363
364
365     Subclasses of this one should re-define the _real_initialize() and
366     _real_extract() methods and define a _VALID_URL regexp.
367     Probably, they should also be added to the list of extractors.
368
369     _GEO_BYPASS attribute may be set to False in order to disable
370     geo restriction bypass mechanisms for a particular extractor.
371     Though it won't disable explicit geo restriction bypass based on
372     country code provided with geo_bypass_country.
373
374     _GEO_COUNTRIES attribute may contain a list of presumably geo unrestricted
375     countries for this extractor. One of these countries will be used by
376     geo restriction bypass mechanism right away in order to bypass
377     geo restriction, of course, if the mechanism is not disabled.
378
379     _GEO_IP_BLOCKS attribute may contain a list of presumably geo unrestricted
380     IP blocks in CIDR notation for this extractor. One of these IP blocks
381     will be used by geo restriction bypass mechanism similarly
382     to _GEO_COUNTRIES.
383
384     Finally, the _WORKING attribute should be set to False for broken IEs
385     in order to warn the users and skip the tests.
386     """
387
388     _ready = False
389     _downloader = None
390     _x_forwarded_for_ip = None
391     _GEO_BYPASS = True
392     _GEO_COUNTRIES = None
393     _GEO_IP_BLOCKS = None
394     _WORKING = True
395
396     def __init__(self, downloader=None):
397         """Constructor. Receives an optional downloader."""
398         self._ready = False
399         self._x_forwarded_for_ip = None
400         self.set_downloader(downloader)
401
402     @classmethod
403     def suitable(cls, url):
404         """Receives a URL and returns True if suitable for this IE."""
405
406         # This does not use has/getattr intentionally - we want to know whether
407         # we have cached the regexp for *this* class, whereas getattr would also
408         # match the superclass
409         if '_VALID_URL_RE' not in cls.__dict__:
410             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
411         return cls._VALID_URL_RE.match(url) is not None
412
413     @classmethod
414     def _match_id(cls, url):
415         if '_VALID_URL_RE' not in cls.__dict__:
416             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
417         m = cls._VALID_URL_RE.match(url)
418         assert m
419         return compat_str(m.group('id'))
420
421     @classmethod
422     def working(cls):
423         """Getter method for _WORKING."""
424         return cls._WORKING
425
426     def initialize(self):
427         """Initializes an instance (authentication, etc)."""
428         self._initialize_geo_bypass({
429             'countries': self._GEO_COUNTRIES,
430             'ip_blocks': self._GEO_IP_BLOCKS,
431         })
432         if not self._ready:
433             self._real_initialize()
434             self._ready = True
435
436     def _initialize_geo_bypass(self, geo_bypass_context):
437         """
438         Initialize geo restriction bypass mechanism.
439
440         This method is used to initialize geo bypass mechanism based on faking
441         X-Forwarded-For HTTP header. A random country from provided country list
442         is selected and a random IP belonging to this country is generated. This
443         IP will be passed as X-Forwarded-For HTTP header in all subsequent
444         HTTP requests.
445
446         This method will be used for initial geo bypass mechanism initialization
447         during the instance initialization with _GEO_COUNTRIES and
448         _GEO_IP_BLOCKS.
449
450         You may also manually call it from extractor's code if geo bypass
451         information is not available beforehand (e.g. obtained during
452         extraction) or due to some other reason. In this case you should pass
453         this information in geo bypass context passed as first argument. It may
454         contain following fields:
455
456         countries:  List of geo unrestricted countries (similar
457                     to _GEO_COUNTRIES)
458         ip_blocks:  List of geo unrestricted IP blocks in CIDR notation
459                     (similar to _GEO_IP_BLOCKS)
460
461         """
462         if not self._x_forwarded_for_ip:
463
464             # Geo bypass mechanism is explicitly disabled by user
465             if not self._downloader.params.get('geo_bypass', True):
466                 return
467
468             if not geo_bypass_context:
469                 geo_bypass_context = {}
470
471             # Backward compatibility: previously _initialize_geo_bypass
472             # expected a list of countries, some 3rd party code may still use
473             # it this way
474             if isinstance(geo_bypass_context, (list, tuple)):
475                 geo_bypass_context = {
476                     'countries': geo_bypass_context,
477                 }
478
479             # The whole point of geo bypass mechanism is to fake IP
480             # as X-Forwarded-For HTTP header based on some IP block or
481             # country code.
482
483             # Path 1: bypassing based on IP block in CIDR notation
484
485             # Explicit IP block specified by user, use it right away
486             # regardless of whether extractor is geo bypassable or not
487             ip_block = self._downloader.params.get('geo_bypass_ip_block', None)
488
489             # Otherwise use random IP block from geo bypass context but only
490             # if extractor is known as geo bypassable
491             if not ip_block:
492                 ip_blocks = geo_bypass_context.get('ip_blocks')
493                 if self._GEO_BYPASS and ip_blocks:
494                     ip_block = random.choice(ip_blocks)
495
496             if ip_block:
497                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block)
498                 if self._downloader.params.get('verbose', False):
499                     self._downloader.to_screen(
500                         '[debug] Using fake IP %s as X-Forwarded-For.'
501                         % self._x_forwarded_for_ip)
502                 return
503
504             # Path 2: bypassing based on country code
505
506             # Explicit country code specified by user, use it right away
507             # regardless of whether extractor is geo bypassable or not
508             country = self._downloader.params.get('geo_bypass_country', None)
509
510             # Otherwise use random country code from geo bypass context but
511             # only if extractor is known as geo bypassable
512             if not country:
513                 countries = geo_bypass_context.get('countries')
514                 if self._GEO_BYPASS and countries:
515                     country = random.choice(countries)
516
517             if country:
518                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country)
519                 if self._downloader.params.get('verbose', False):
520                     self._downloader.to_screen(
521                         '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
522                         % (self._x_forwarded_for_ip, country.upper()))
523
524     def extract(self, url):
525         """Extracts URL information and returns it in list of dicts."""
526         try:
527             for _ in range(2):
528                 try:
529                     self.initialize()
530                     ie_result = self._real_extract(url)
531                     if self._x_forwarded_for_ip:
532                         ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
533                     return ie_result
534                 except GeoRestrictedError as e:
535                     if self.__maybe_fake_ip_and_retry(e.countries):
536                         continue
537                     raise
538         except ExtractorError:
539             raise
540         except compat_http_client.IncompleteRead as e:
541             raise ExtractorError('A network error has occurred.', cause=e, expected=True)
542         except (KeyError, StopIteration) as e:
543             raise ExtractorError('An extractor error has occurred.', cause=e)
544
545     def __maybe_fake_ip_and_retry(self, countries):
546         if (not self._downloader.params.get('geo_bypass_country', None)
547                 and self._GEO_BYPASS
548                 and self._downloader.params.get('geo_bypass', True)
549                 and not self._x_forwarded_for_ip
550                 and countries):
551             country_code = random.choice(countries)
552             self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
553             if self._x_forwarded_for_ip:
554                 self.report_warning(
555                     'Video is geo restricted. Retrying extraction with fake IP %s (%s) as X-Forwarded-For.'
556                     % (self._x_forwarded_for_ip, country_code.upper()))
557                 return True
558         return False
559
560     def set_downloader(self, downloader):
561         """Sets the downloader for this IE."""
562         self._downloader = downloader
563
564     def _real_initialize(self):
565         """Real initialization process. Redefine in subclasses."""
566         pass
567
568     def _real_extract(self, url):
569         """Real extraction process. Redefine in subclasses."""
570         pass
571
572     @classmethod
573     def ie_key(cls):
574         """A string for getting the InfoExtractor with get_info_extractor"""
575         return compat_str(cls.__name__[:-2])
576
577     @property
578     def IE_NAME(self):
579         return compat_str(type(self).__name__[:-2])
580
581     @staticmethod
582     def __can_accept_status_code(err, expected_status):
583         assert isinstance(err, compat_urllib_error.HTTPError)
584         if expected_status is None:
585             return False
586         if isinstance(expected_status, compat_integer_types):
587             return err.code == expected_status
588         elif isinstance(expected_status, (list, tuple)):
589             return err.code in expected_status
590         elif callable(expected_status):
591             return expected_status(err.code) is True
592         else:
593             assert False
594
595     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}, expected_status=None):
596         """
597         Return the response handle.
598
599         See _download_webpage docstring for arguments specification.
600         """
601         if note is None:
602             self.report_download_webpage(video_id)
603         elif note is not False:
604             if video_id is None:
605                 self.to_screen('%s' % (note,))
606             else:
607                 self.to_screen('%s: %s' % (video_id, note))
608
609         # Some sites check X-Forwarded-For HTTP header in order to figure out
610         # the origin of the client behind proxy. This allows bypassing geo
611         # restriction by faking this header's value to IP that belongs to some
612         # geo unrestricted country. We will do so once we encounter any
613         # geo restriction error.
614         if self._x_forwarded_for_ip:
615             if 'X-Forwarded-For' not in headers:
616                 headers['X-Forwarded-For'] = self._x_forwarded_for_ip
617
618         if isinstance(url_or_request, compat_urllib_request.Request):
619             url_or_request = update_Request(
620                 url_or_request, data=data, headers=headers, query=query)
621         else:
622             if query:
623                 url_or_request = update_url_query(url_or_request, query)
624             if data is not None or headers:
625                 url_or_request = sanitized_Request(url_or_request, data, headers)
626         try:
627             return self._downloader.urlopen(url_or_request)
628         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
629             if isinstance(err, compat_urllib_error.HTTPError):
630                 if self.__can_accept_status_code(err, expected_status):
631                     # Retain reference to error to prevent file object from
632                     # being closed before it can be read. Works around the
633                     # effects of <https://bugs.python.org/issue15002>
634                     # introduced in Python 3.4.1.
635                     err.fp._error = err
636                     return err.fp
637
638             if errnote is False:
639                 return False
640             if errnote is None:
641                 errnote = 'Unable to download webpage'
642
643             errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
644             if fatal:
645                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
646             else:
647                 self._downloader.report_warning(errmsg)
648                 return False
649
650     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
651         """
652         Return a tuple (page content as string, URL handle).
653
654         See _download_webpage docstring for arguments specification.
655         """
656         # Strip hashes from the URL (#1038)
657         if isinstance(url_or_request, (compat_str, str)):
658             url_or_request = url_or_request.partition('#')[0]
659
660         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query, expected_status=expected_status)
661         if urlh is False:
662             assert not fatal
663             return False
664         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
665         return (content, urlh)
666
667     @staticmethod
668     def _guess_encoding_from_content(content_type, webpage_bytes):
669         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
670         if m:
671             encoding = m.group(1)
672         else:
673             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
674                           webpage_bytes[:1024])
675             if m:
676                 encoding = m.group(1).decode('ascii')
677             elif webpage_bytes.startswith(b'\xff\xfe'):
678                 encoding = 'utf-16'
679             else:
680                 encoding = 'utf-8'
681
682         return encoding
683
684     def __check_blocked(self, content):
685         first_block = content[:512]
686         if ('<title>Access to this site is blocked</title>' in content
687                 and 'Websense' in first_block):
688             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
689             blocked_iframe = self._html_search_regex(
690                 r'<iframe src="([^"]+)"', content,
691                 'Websense information URL', default=None)
692             if blocked_iframe:
693                 msg += ' Visit %s for more details' % blocked_iframe
694             raise ExtractorError(msg, expected=True)
695         if '<title>The URL you requested has been blocked</title>' in first_block:
696             msg = (
697                 'Access to this webpage has been blocked by Indian censorship. '
698                 'Use a VPN or proxy server (with --proxy) to route around it.')
699             block_msg = self._html_search_regex(
700                 r'</h1><p>(.*?)</p>',
701                 content, 'block message', default=None)
702             if block_msg:
703                 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
704             raise ExtractorError(msg, expected=True)
705         if ('<title>TTK :: Доступ к ресурсу ограничен</title>' in content
706                 and 'blocklist.rkn.gov.ru' in content):
707             raise ExtractorError(
708                 'Access to this webpage has been blocked by decision of the Russian government. '
709                 'Visit http://blocklist.rkn.gov.ru/ for a block reason.',
710                 expected=True)
711
712     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
713         content_type = urlh.headers.get('Content-Type', '')
714         webpage_bytes = urlh.read()
715         if prefix is not None:
716             webpage_bytes = prefix + webpage_bytes
717         if not encoding:
718             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
719         if self._downloader.params.get('dump_intermediate_pages', False):
720             self.to_screen('Dumping request to ' + urlh.geturl())
721             dump = base64.b64encode(webpage_bytes).decode('ascii')
722             self._downloader.to_screen(dump)
723         if self._downloader.params.get('write_pages', False):
724             basen = '%s_%s' % (video_id, urlh.geturl())
725             if len(basen) > 240:
726                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
727                 basen = basen[:240 - len(h)] + h
728             raw_filename = basen + '.dump'
729             filename = sanitize_filename(raw_filename, restricted=True)
730             self.to_screen('Saving request to ' + filename)
731             # Working around MAX_PATH limitation on Windows (see
732             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
733             if compat_os_name == 'nt':
734                 absfilepath = os.path.abspath(filename)
735                 if len(absfilepath) > 259:
736                     filename = '\\\\?\\' + absfilepath
737             with open(filename, 'wb') as outf:
738                 outf.write(webpage_bytes)
739
740         try:
741             content = webpage_bytes.decode(encoding, 'replace')
742         except LookupError:
743             content = webpage_bytes.decode('utf-8', 'replace')
744
745         self.__check_blocked(content)
746
747         return content
748
749     def _download_webpage(
750             self, url_or_request, video_id, note=None, errnote=None,
751             fatal=True, tries=1, timeout=5, encoding=None, data=None,
752             headers={}, query={}, expected_status=None):
753         """
754         Return the data of the page as a string.
755
756         Arguments:
757         url_or_request -- plain text URL as a string or
758             a compat_urllib_request.Requestobject
759         video_id -- Video/playlist/item identifier (string)
760
761         Keyword arguments:
762         note -- note printed before downloading (string)
763         errnote -- note printed in case of an error (string)
764         fatal -- flag denoting whether error should be considered fatal,
765             i.e. whether it should cause ExtractionError to be raised,
766             otherwise a warning will be reported and extraction continued
767         tries -- number of tries
768         timeout -- sleep interval between tries
769         encoding -- encoding for a page content decoding, guessed automatically
770             when not explicitly specified
771         data -- POST data (bytes)
772         headers -- HTTP headers (dict)
773         query -- URL query (dict)
774         expected_status -- allows to accept failed HTTP requests (non 2xx
775             status code) by explicitly specifying a set of accepted status
776             codes. Can be any of the following entities:
777                 - an integer type specifying an exact failed status code to
778                   accept
779                 - a list or a tuple of integer types specifying a list of
780                   failed status codes to accept
781                 - a callable accepting an actual failed status code and
782                   returning True if it should be accepted
783             Note that this argument does not affect success status codes (2xx)
784             which are always accepted.
785         """
786
787         success = False
788         try_count = 0
789         while success is False:
790             try:
791                 res = self._download_webpage_handle(
792                     url_or_request, video_id, note, errnote, fatal,
793                     encoding=encoding, data=data, headers=headers, query=query,
794                     expected_status=expected_status)
795                 success = True
796             except compat_http_client.IncompleteRead as e:
797                 try_count += 1
798                 if try_count >= tries:
799                     raise e
800                 self._sleep(timeout, video_id)
801         if res is False:
802             return res
803         else:
804             content, _ = res
805             return content
806
807     def _download_xml_handle(
808             self, url_or_request, video_id, note='Downloading XML',
809             errnote='Unable to download XML', transform_source=None,
810             fatal=True, encoding=None, data=None, headers={}, query={},
811             expected_status=None):
812         """
813         Return a tuple (xml as an compat_etree_Element, URL handle).
814
815         See _download_webpage docstring for arguments specification.
816         """
817         res = self._download_webpage_handle(
818             url_or_request, video_id, note, errnote, fatal=fatal,
819             encoding=encoding, data=data, headers=headers, query=query,
820             expected_status=expected_status)
821         if res is False:
822             return res
823         xml_string, urlh = res
824         return self._parse_xml(
825             xml_string, video_id, transform_source=transform_source,
826             fatal=fatal), urlh
827
828     def _download_xml(
829             self, url_or_request, video_id,
830             note='Downloading XML', errnote='Unable to download XML',
831             transform_source=None, fatal=True, encoding=None,
832             data=None, headers={}, query={}, expected_status=None):
833         """
834         Return the xml as an compat_etree_Element.
835
836         See _download_webpage docstring for arguments specification.
837         """
838         res = self._download_xml_handle(
839             url_or_request, video_id, note=note, errnote=errnote,
840             transform_source=transform_source, fatal=fatal, encoding=encoding,
841             data=data, headers=headers, query=query,
842             expected_status=expected_status)
843         return res if res is False else res[0]
844
845     def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
846         if transform_source:
847             xml_string = transform_source(xml_string)
848         try:
849             return compat_etree_fromstring(xml_string.encode('utf-8'))
850         except compat_xml_parse_error as ve:
851             errmsg = '%s: Failed to parse XML ' % video_id
852             if fatal:
853                 raise ExtractorError(errmsg, cause=ve)
854             else:
855                 self.report_warning(errmsg + str(ve))
856
857     def _download_json_handle(
858             self, url_or_request, video_id, note='Downloading JSON metadata',
859             errnote='Unable to download JSON metadata', transform_source=None,
860             fatal=True, encoding=None, data=None, headers={}, query={},
861             expected_status=None):
862         """
863         Return a tuple (JSON object, URL handle).
864
865         See _download_webpage docstring for arguments specification.
866         """
867         res = self._download_webpage_handle(
868             url_or_request, video_id, note, errnote, fatal=fatal,
869             encoding=encoding, data=data, headers=headers, query=query,
870             expected_status=expected_status)
871         if res is False:
872             return res
873         json_string, urlh = res
874         return self._parse_json(
875             json_string, video_id, transform_source=transform_source,
876             fatal=fatal), urlh
877
878     def _download_json(
879             self, url_or_request, video_id, note='Downloading JSON metadata',
880             errnote='Unable to download JSON metadata', transform_source=None,
881             fatal=True, encoding=None, data=None, headers={}, query={},
882             expected_status=None):
883         """
884         Return the JSON object as a dict.
885
886         See _download_webpage docstring for arguments specification.
887         """
888         res = self._download_json_handle(
889             url_or_request, video_id, note=note, errnote=errnote,
890             transform_source=transform_source, fatal=fatal, encoding=encoding,
891             data=data, headers=headers, query=query,
892             expected_status=expected_status)
893         return res if res is False else res[0]
894
895     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
896         if transform_source:
897             json_string = transform_source(json_string)
898         try:
899             return json.loads(json_string)
900         except ValueError as ve:
901             errmsg = '%s: Failed to parse JSON ' % video_id
902             if fatal:
903                 raise ExtractorError(errmsg, cause=ve)
904             else:
905                 self.report_warning(errmsg + str(ve))
906
907     def report_warning(self, msg, video_id=None):
908         idstr = '' if video_id is None else '%s: ' % video_id
909         self._downloader.report_warning(
910             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
911
912     def to_screen(self, msg):
913         """Print msg to screen, prefixing it with '[ie_name]'"""
914         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
915
916     def report_extraction(self, id_or_name):
917         """Report information extraction."""
918         self.to_screen('%s: Extracting information' % id_or_name)
919
920     def report_download_webpage(self, video_id):
921         """Report webpage download."""
922         self.to_screen('%s: Downloading webpage' % video_id)
923
924     def report_age_confirmation(self):
925         """Report attempt to confirm age."""
926         self.to_screen('Confirming age')
927
928     def report_login(self):
929         """Report attempt to log in."""
930         self.to_screen('Logging in')
931
932     @staticmethod
933     def raise_login_required(msg='This video is only available for registered users'):
934         raise ExtractorError(
935             '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
936             expected=True)
937
938     @staticmethod
939     def raise_geo_restricted(msg='This video is not available from your location due to geo restriction', countries=None):
940         raise GeoRestrictedError(msg, countries=countries)
941
942     # Methods for following #608
943     @staticmethod
944     def url_result(url, ie=None, video_id=None, video_title=None):
945         """Returns a URL that points to a page that should be processed"""
946         # TODO: ie should be the class used for getting the info
947         video_info = {'_type': 'url',
948                       'url': url,
949                       'ie_key': ie}
950         if video_id is not None:
951             video_info['id'] = video_id
952         if video_title is not None:
953             video_info['title'] = video_title
954         return video_info
955
956     def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None):
957         urls = orderedSet(
958             self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
959             for m in matches)
960         return self.playlist_result(
961             urls, playlist_id=playlist_id, playlist_title=playlist_title)
962
963     @staticmethod
964     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
965         """Returns a playlist"""
966         video_info = {'_type': 'playlist',
967                       'entries': entries}
968         if playlist_id:
969             video_info['id'] = playlist_id
970         if playlist_title:
971             video_info['title'] = playlist_title
972         if playlist_description:
973             video_info['description'] = playlist_description
974         return video_info
975
976     def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
977         """
978         Perform a regex search on the given string, using a single or a list of
979         patterns returning the first matching group.
980         In case of failure return a default value or raise a WARNING or a
981         RegexNotFoundError, depending on fatal, specifying the field name.
982         """
983         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
984             mobj = re.search(pattern, string, flags)
985         else:
986             for p in pattern:
987                 mobj = re.search(p, string, flags)
988                 if mobj:
989                     break
990
991         if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
992             _name = '\033[0;34m%s\033[0m' % name
993         else:
994             _name = name
995
996         if mobj:
997             if group is None:
998                 # return the first matching group
999                 return next(g for g in mobj.groups() if g is not None)
1000             else:
1001                 return mobj.group(group)
1002         elif default is not NO_DEFAULT:
1003             return default
1004         elif fatal:
1005             raise RegexNotFoundError('Unable to extract %s' % _name)
1006         else:
1007             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
1008             return None
1009
1010     def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
1011         """
1012         Like _search_regex, but strips HTML tags and unescapes entities.
1013         """
1014         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
1015         if res:
1016             return clean_html(res).strip()
1017         else:
1018             return res
1019
1020     def _get_netrc_login_info(self, netrc_machine=None):
1021         username = None
1022         password = None
1023         netrc_machine = netrc_machine or self._NETRC_MACHINE
1024
1025         if self._downloader.params.get('usenetrc', False):
1026             try:
1027                 info = netrc.netrc().authenticators(netrc_machine)
1028                 if info is not None:
1029                     username = info[0]
1030                     password = info[2]
1031                 else:
1032                     raise netrc.NetrcParseError(
1033                         'No authenticators for %s' % netrc_machine)
1034             except (IOError, netrc.NetrcParseError) as err:
1035                 self._downloader.report_warning(
1036                     'parsing .netrc: %s' % error_to_compat_str(err))
1037
1038         return username, password
1039
1040     def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
1041         """
1042         Get the login info as (username, password)
1043         First look for the manually specified credentials using username_option
1044         and password_option as keys in params dictionary. If no such credentials
1045         available look in the netrc file using the netrc_machine or _NETRC_MACHINE
1046         value.
1047         If there's no info available, return (None, None)
1048         """
1049         if self._downloader is None:
1050             return (None, None)
1051
1052         downloader_params = self._downloader.params
1053
1054         # Attempt to use provided username and password or .netrc data
1055         if downloader_params.get(username_option) is not None:
1056             username = downloader_params[username_option]
1057             password = downloader_params[password_option]
1058         else:
1059             username, password = self._get_netrc_login_info(netrc_machine)
1060
1061         return username, password
1062
1063     def _get_tfa_info(self, note='two-factor verification code'):
1064         """
1065         Get the two-factor authentication info
1066         TODO - asking the user will be required for sms/phone verify
1067         currently just uses the command line option
1068         If there's no info available, return None
1069         """
1070         if self._downloader is None:
1071             return None
1072         downloader_params = self._downloader.params
1073
1074         if downloader_params.get('twofactor') is not None:
1075             return downloader_params['twofactor']
1076
1077         return compat_getpass('Type %s and press [Return]: ' % note)
1078
1079     # Helper functions for extracting OpenGraph info
1080     @staticmethod
1081     def _og_regexes(prop):
1082         content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
1083         property_re = (r'(?:name|property)=(?:\'og[:-]%(prop)s\'|"og[:-]%(prop)s"|\s*og[:-]%(prop)s\b)'
1084                        % {'prop': re.escape(prop)})
1085         template = r'<meta[^>]+?%s[^>]+?%s'
1086         return [
1087             template % (property_re, content_re),
1088             template % (content_re, property_re),
1089         ]
1090
1091     @staticmethod
1092     def _meta_regex(prop):
1093         return r'''(?isx)<meta
1094                     (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
1095                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
1096
1097     def _og_search_property(self, prop, html, name=None, **kargs):
1098         if not isinstance(prop, (list, tuple)):
1099             prop = [prop]
1100         if name is None:
1101             name = 'OpenGraph %s' % prop[0]
1102         og_regexes = []
1103         for p in prop:
1104             og_regexes.extend(self._og_regexes(p))
1105         escaped = self._search_regex(og_regexes, html, name, flags=re.DOTALL, **kargs)
1106         if escaped is None:
1107             return None
1108         return unescapeHTML(escaped)
1109
1110     def _og_search_thumbnail(self, html, **kargs):
1111         return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
1112
1113     def _og_search_description(self, html, **kargs):
1114         return self._og_search_property('description', html, fatal=False, **kargs)
1115
1116     def _og_search_title(self, html, **kargs):
1117         return self._og_search_property('title', html, **kargs)
1118
1119     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
1120         regexes = self._og_regexes('video') + self._og_regexes('video:url')
1121         if secure:
1122             regexes = self._og_regexes('video:secure_url') + regexes
1123         return self._html_search_regex(regexes, html, name, **kargs)
1124
1125     def _og_search_url(self, html, **kargs):
1126         return self._og_search_property('url', html, **kargs)
1127
1128     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
1129         if not isinstance(name, (list, tuple)):
1130             name = [name]
1131         if display_name is None:
1132             display_name = name[0]
1133         return self._html_search_regex(
1134             [self._meta_regex(n) for n in name],
1135             html, display_name, fatal=fatal, group='content', **kwargs)
1136
1137     def _dc_search_uploader(self, html):
1138         return self._html_search_meta('dc.creator', html, 'uploader')
1139
1140     def _rta_search(self, html):
1141         # See http://www.rtalabel.org/index.php?content=howtofaq#single
1142         if re.search(r'(?ix)<meta\s+name="rating"\s+'
1143                      r'     content="RTA-5042-1996-1400-1577-RTA"',
1144                      html):
1145             return 18
1146         return 0
1147
1148     def _media_rating_search(self, html):
1149         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
1150         rating = self._html_search_meta('rating', html)
1151
1152         if not rating:
1153             return None
1154
1155         RATING_TABLE = {
1156             'safe for kids': 0,
1157             'general': 8,
1158             '14 years': 14,
1159             'mature': 17,
1160             'restricted': 19,
1161         }
1162         return RATING_TABLE.get(rating.lower())
1163
1164     def _family_friendly_search(self, html):
1165         # See http://schema.org/VideoObject
1166         family_friendly = self._html_search_meta(
1167             'isFamilyFriendly', html, default=None)
1168
1169         if not family_friendly:
1170             return None
1171
1172         RATING_TABLE = {
1173             '1': 0,
1174             'true': 0,
1175             '0': 18,
1176             'false': 18,
1177         }
1178         return RATING_TABLE.get(family_friendly.lower())
1179
1180     def _twitter_search_player(self, html):
1181         return self._html_search_meta('twitter:player', html,
1182                                       'twitter card player')
1183
1184     def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
1185         json_ld = self._search_regex(
1186             JSON_LD_RE, html, 'JSON-LD', group='json_ld', **kwargs)
1187         default = kwargs.get('default', NO_DEFAULT)
1188         if not json_ld:
1189             return default if default is not NO_DEFAULT else {}
1190         # JSON-LD may be malformed and thus `fatal` should be respected.
1191         # At the same time `default` may be passed that assumes `fatal=False`
1192         # for _search_regex. Let's simulate the same behavior here as well.
1193         fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
1194         return self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)
1195
1196     def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
1197         if isinstance(json_ld, compat_str):
1198             json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
1199         if not json_ld:
1200             return {}
1201         info = {}
1202         if not isinstance(json_ld, (list, tuple, dict)):
1203             return info
1204         if isinstance(json_ld, dict):
1205             json_ld = [json_ld]
1206
1207         INTERACTION_TYPE_MAP = {
1208             'CommentAction': 'comment',
1209             'AgreeAction': 'like',
1210             'DisagreeAction': 'dislike',
1211             'LikeAction': 'like',
1212             'DislikeAction': 'dislike',
1213             'ListenAction': 'view',
1214             'WatchAction': 'view',
1215             'ViewAction': 'view',
1216         }
1217
1218         def extract_interaction_statistic(e):
1219             interaction_statistic = e.get('interactionStatistic')
1220             if not isinstance(interaction_statistic, list):
1221                 return
1222             for is_e in interaction_statistic:
1223                 if not isinstance(is_e, dict):
1224                     continue
1225                 if is_e.get('@type') != 'InteractionCounter':
1226                     continue
1227                 interaction_type = is_e.get('interactionType')
1228                 if not isinstance(interaction_type, compat_str):
1229                     continue
1230                 interaction_count = int_or_none(is_e.get('userInteractionCount'))
1231                 if interaction_count is None:
1232                     continue
1233                 count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1])
1234                 if not count_kind:
1235                     continue
1236                 count_key = '%s_count' % count_kind
1237                 if info.get(count_key) is not None:
1238                     continue
1239                 info[count_key] = interaction_count
1240
1241         def extract_video_object(e):
1242             assert e['@type'] == 'VideoObject'
1243             info.update({
1244                 'url': url_or_none(e.get('contentUrl')),
1245                 'title': unescapeHTML(e.get('name')),
1246                 'description': unescapeHTML(e.get('description')),
1247                 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
1248                 'duration': parse_duration(e.get('duration')),
1249                 'timestamp': unified_timestamp(e.get('uploadDate')),
1250                 'filesize': float_or_none(e.get('contentSize')),
1251                 'tbr': int_or_none(e.get('bitrate')),
1252                 'width': int_or_none(e.get('width')),
1253                 'height': int_or_none(e.get('height')),
1254                 'view_count': int_or_none(e.get('interactionCount')),
1255             })
1256             extract_interaction_statistic(e)
1257
1258         for e in json_ld:
1259             if isinstance(e.get('@context'), compat_str) and re.match(r'^https?://schema.org/?$', e.get('@context')):
1260                 item_type = e.get('@type')
1261                 if expected_type is not None and expected_type != item_type:
1262                     return info
1263                 if item_type in ('TVEpisode', 'Episode'):
1264                     episode_name = unescapeHTML(e.get('name'))
1265                     info.update({
1266                         'episode': episode_name,
1267                         'episode_number': int_or_none(e.get('episodeNumber')),
1268                         'description': unescapeHTML(e.get('description')),
1269                     })
1270                     if not info.get('title') and episode_name:
1271                         info['title'] = episode_name
1272                     part_of_season = e.get('partOfSeason')
1273                     if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
1274                         info.update({
1275                             'season': unescapeHTML(part_of_season.get('name')),
1276                             'season_number': int_or_none(part_of_season.get('seasonNumber')),
1277                         })
1278                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
1279                     if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
1280                         info['series'] = unescapeHTML(part_of_series.get('name'))
1281                 elif item_type == 'Movie':
1282                     info.update({
1283                         'title': unescapeHTML(e.get('name')),
1284                         'description': unescapeHTML(e.get('description')),
1285                         'duration': parse_duration(e.get('duration')),
1286                         'timestamp': unified_timestamp(e.get('dateCreated')),
1287                     })
1288                 elif item_type in ('Article', 'NewsArticle'):
1289                     info.update({
1290                         'timestamp': parse_iso8601(e.get('datePublished')),
1291                         'title': unescapeHTML(e.get('headline')),
1292                         'description': unescapeHTML(e.get('articleBody')),
1293                     })
1294                 elif item_type == 'VideoObject':
1295                     extract_video_object(e)
1296                     continue
1297                 video = e.get('video')
1298                 if isinstance(video, dict) and video.get('@type') == 'VideoObject':
1299                     extract_video_object(video)
1300                 break
1301         return dict((k, v) for k, v in info.items() if v is not None)
1302
1303     @staticmethod
1304     def _hidden_inputs(html):
1305         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
1306         hidden_inputs = {}
1307         for input in re.findall(r'(?i)(<input[^>]+>)', html):
1308             attrs = extract_attributes(input)
1309             if not input:
1310                 continue
1311             if attrs.get('type') not in ('hidden', 'submit'):
1312                 continue
1313             name = attrs.get('name') or attrs.get('id')
1314             value = attrs.get('value')
1315             if name and value is not None:
1316                 hidden_inputs[name] = value
1317         return hidden_inputs
1318
1319     def _form_hidden_inputs(self, form_id, html):
1320         form = self._search_regex(
1321             r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
1322             html, '%s form' % form_id, group='form')
1323         return self._hidden_inputs(form)
1324
1325     def _sort_formats(self, formats, field_preference=None):
1326         if not formats:
1327             raise ExtractorError('No video formats found')
1328
1329         for f in formats:
1330             # Automatically determine tbr when missing based on abr and vbr (improves
1331             # formats sorting in some cases)
1332             if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
1333                 f['tbr'] = f['abr'] + f['vbr']
1334
1335         def _formats_key(f):
1336             # TODO remove the following workaround
1337             from ..utils import determine_ext
1338             if not f.get('ext') and 'url' in f:
1339                 f['ext'] = determine_ext(f['url'])
1340
1341             if isinstance(field_preference, (list, tuple)):
1342                 return tuple(
1343                     f.get(field)
1344                     if f.get(field) is not None
1345                     else ('' if field == 'format_id' else -1)
1346                     for field in field_preference)
1347
1348             preference = f.get('preference')
1349             if preference is None:
1350                 preference = 0
1351                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
1352                     preference -= 0.5
1353
1354             protocol = f.get('protocol') or determine_protocol(f)
1355             proto_preference = 0 if protocol in ['http', 'https'] else (-0.5 if protocol == 'rtsp' else -0.1)
1356
1357             if f.get('vcodec') == 'none':  # audio only
1358                 preference -= 50
1359                 if self._downloader.params.get('prefer_free_formats'):
1360                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
1361                 else:
1362                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
1363                 ext_preference = 0
1364                 try:
1365                     audio_ext_preference = ORDER.index(f['ext'])
1366                 except ValueError:
1367                     audio_ext_preference = -1
1368             else:
1369                 if f.get('acodec') == 'none':  # video only
1370                     preference -= 40
1371                 if self._downloader.params.get('prefer_free_formats'):
1372                     ORDER = ['flv', 'mp4', 'webm']
1373                 else:
1374                     ORDER = ['webm', 'flv', 'mp4']
1375                 try:
1376                     ext_preference = ORDER.index(f['ext'])
1377                 except ValueError:
1378                     ext_preference = -1
1379                 audio_ext_preference = 0
1380
1381             return (
1382                 preference,
1383                 f.get('language_preference') if f.get('language_preference') is not None else -1,
1384                 f.get('quality') if f.get('quality') is not None else -1,
1385                 f.get('tbr') if f.get('tbr') is not None else -1,
1386                 f.get('filesize') if f.get('filesize') is not None else -1,
1387                 f.get('vbr') if f.get('vbr') is not None else -1,
1388                 f.get('height') if f.get('height') is not None else -1,
1389                 f.get('width') if f.get('width') is not None else -1,
1390                 proto_preference,
1391                 ext_preference,
1392                 f.get('abr') if f.get('abr') is not None else -1,
1393                 audio_ext_preference,
1394                 f.get('fps') if f.get('fps') is not None else -1,
1395                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
1396                 f.get('source_preference') if f.get('source_preference') is not None else -1,
1397                 f.get('format_id') if f.get('format_id') is not None else '',
1398             )
1399         formats.sort(key=_formats_key)
1400
1401     def _check_formats(self, formats, video_id):
1402         if formats:
1403             formats[:] = filter(
1404                 lambda f: self._is_valid_url(
1405                     f['url'], video_id,
1406                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
1407                 formats)
1408
1409     @staticmethod
1410     def _remove_duplicate_formats(formats):
1411         format_urls = set()
1412         unique_formats = []
1413         for f in formats:
1414             if f['url'] not in format_urls:
1415                 format_urls.add(f['url'])
1416                 unique_formats.append(f)
1417         formats[:] = unique_formats
1418
1419     def _is_valid_url(self, url, video_id, item='video', headers={}):
1420         url = self._proto_relative_url(url, scheme='http:')
1421         # For now assume non HTTP(S) URLs always valid
1422         if not (url.startswith('http://') or url.startswith('https://')):
1423             return True
1424         try:
1425             self._request_webpage(url, video_id, 'Checking %s URL' % item, headers=headers)
1426             return True
1427         except ExtractorError:
1428             self.to_screen(
1429                 '%s: %s URL is invalid, skipping' % (video_id, item))
1430             return False
1431
1432     def http_scheme(self):
1433         """ Either "http:" or "https:", depending on the user's preferences """
1434         return (
1435             'http:'
1436             if self._downloader.params.get('prefer_insecure', False)
1437             else 'https:')
1438
1439     def _proto_relative_url(self, url, scheme=None):
1440         if url is None:
1441             return url
1442         if url.startswith('//'):
1443             if scheme is None:
1444                 scheme = self.http_scheme()
1445             return scheme + url
1446         else:
1447             return url
1448
1449     def _sleep(self, timeout, video_id, msg_template=None):
1450         if msg_template is None:
1451             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
1452         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
1453         self.to_screen(msg)
1454         time.sleep(timeout)
1455
1456     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
1457                              transform_source=lambda s: fix_xml_ampersands(s).strip(),
1458                              fatal=True, m3u8_id=None, data=None, headers={}, query={}):
1459         manifest = self._download_xml(
1460             manifest_url, video_id, 'Downloading f4m manifest',
1461             'Unable to download f4m manifest',
1462             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
1463             # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244)
1464             transform_source=transform_source,
1465             fatal=fatal, data=data, headers=headers, query=query)
1466
1467         if manifest is False:
1468             return []
1469
1470         return self._parse_f4m_formats(
1471             manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1472             transform_source=transform_source, fatal=fatal, m3u8_id=m3u8_id)
1473
1474     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
1475                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
1476                            fatal=True, m3u8_id=None):
1477         if not isinstance(manifest, compat_etree_Element) and not fatal:
1478             return []
1479
1480         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
1481         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
1482         if akamai_pv is not None and ';' in akamai_pv.text:
1483             playerVerificationChallenge = akamai_pv.text.split(';')[0]
1484             if playerVerificationChallenge.strip() != '':
1485                 return []
1486
1487         formats = []
1488         manifest_version = '1.0'
1489         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
1490         if not media_nodes:
1491             manifest_version = '2.0'
1492             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
1493         # Remove unsupported DRM protected media from final formats
1494         # rendition (see https://github.com/ytdl-org/youtube-dl/issues/8573).
1495         media_nodes = remove_encrypted_media(media_nodes)
1496         if not media_nodes:
1497             return formats
1498
1499         manifest_base_url = get_base_url(manifest)
1500
1501         bootstrap_info = xpath_element(
1502             manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
1503             'bootstrap info', default=None)
1504
1505         vcodec = None
1506         mime_type = xpath_text(
1507             manifest, ['{http://ns.adobe.com/f4m/1.0}mimeType', '{http://ns.adobe.com/f4m/2.0}mimeType'],
1508             'base URL', default=None)
1509         if mime_type and mime_type.startswith('audio/'):
1510             vcodec = 'none'
1511
1512         for i, media_el in enumerate(media_nodes):
1513             tbr = int_or_none(media_el.attrib.get('bitrate'))
1514             width = int_or_none(media_el.attrib.get('width'))
1515             height = int_or_none(media_el.attrib.get('height'))
1516             format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
1517             # If <bootstrapInfo> is present, the specified f4m is a
1518             # stream-level manifest, and only set-level manifests may refer to
1519             # external resources.  See section 11.4 and section 4 of F4M spec
1520             if bootstrap_info is None:
1521                 media_url = None
1522                 # @href is introduced in 2.0, see section 11.6 of F4M spec
1523                 if manifest_version == '2.0':
1524                     media_url = media_el.attrib.get('href')
1525                 if media_url is None:
1526                     media_url = media_el.attrib.get('url')
1527                 if not media_url:
1528                     continue
1529                 manifest_url = (
1530                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
1531                     else ((manifest_base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
1532                 # If media_url is itself a f4m manifest do the recursive extraction
1533                 # since bitrates in parent manifest (this one) and media_url manifest
1534                 # may differ leading to inability to resolve the format by requested
1535                 # bitrate in f4m downloader
1536                 ext = determine_ext(manifest_url)
1537                 if ext == 'f4m':
1538                     f4m_formats = self._extract_f4m_formats(
1539                         manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1540                         transform_source=transform_source, fatal=fatal)
1541                     # Sometimes stream-level manifest contains single media entry that
1542                     # does not contain any quality metadata (e.g. http://matchtv.ru/#live-player).
1543                     # At the same time parent's media entry in set-level manifest may
1544                     # contain it. We will copy it from parent in such cases.
1545                     if len(f4m_formats) == 1:
1546                         f = f4m_formats[0]
1547                         f.update({
1548                             'tbr': f.get('tbr') or tbr,
1549                             'width': f.get('width') or width,
1550                             'height': f.get('height') or height,
1551                             'format_id': f.get('format_id') if not tbr else format_id,
1552                             'vcodec': vcodec,
1553                         })
1554                     formats.extend(f4m_formats)
1555                     continue
1556                 elif ext == 'm3u8':
1557                     formats.extend(self._extract_m3u8_formats(
1558                         manifest_url, video_id, 'mp4', preference=preference,
1559                         m3u8_id=m3u8_id, fatal=fatal))
1560                     continue
1561             formats.append({
1562                 'format_id': format_id,
1563                 'url': manifest_url,
1564                 'manifest_url': manifest_url,
1565                 'ext': 'flv' if bootstrap_info is not None else None,
1566                 'protocol': 'f4m',
1567                 'tbr': tbr,
1568                 'width': width,
1569                 'height': height,
1570                 'vcodec': vcodec,
1571                 'preference': preference,
1572             })
1573         return formats
1574
1575     def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, m3u8_id=None):
1576         return {
1577             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
1578             'url': m3u8_url,
1579             'ext': ext,
1580             'protocol': 'm3u8',
1581             'preference': preference - 100 if preference else -100,
1582             'resolution': 'multiple',
1583             'format_note': 'Quality selection URL',
1584         }
1585
1586     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
1587                               entry_protocol='m3u8', preference=None,
1588                               m3u8_id=None, note=None, errnote=None,
1589                               fatal=True, live=False, data=None, headers={},
1590                               query={}):
1591         res = self._download_webpage_handle(
1592             m3u8_url, video_id,
1593             note=note or 'Downloading m3u8 information',
1594             errnote=errnote or 'Failed to download m3u8 information',
1595             fatal=fatal, data=data, headers=headers, query=query)
1596
1597         if res is False:
1598             return []
1599
1600         m3u8_doc, urlh = res
1601         m3u8_url = urlh.geturl()
1602
1603         return self._parse_m3u8_formats(
1604             m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
1605             preference=preference, m3u8_id=m3u8_id, live=live)
1606
1607     def _parse_m3u8_formats(self, m3u8_doc, m3u8_url, ext=None,
1608                             entry_protocol='m3u8', preference=None,
1609                             m3u8_id=None, live=False):
1610         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
1611             return []
1612
1613         if re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc):  # Apple FairPlay
1614             return []
1615
1616         formats = []
1617
1618         format_url = lambda u: (
1619             u
1620             if re.match(r'^https?://', u)
1621             else compat_urlparse.urljoin(m3u8_url, u))
1622
1623         # References:
1624         # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
1625         # 2. https://github.com/ytdl-org/youtube-dl/issues/12211
1626         # 3. https://github.com/ytdl-org/youtube-dl/issues/18923
1627
1628         # We should try extracting formats only from master playlists [1, 4.3.4],
1629         # i.e. playlists that describe available qualities. On the other hand
1630         # media playlists [1, 4.3.3] should be returned as is since they contain
1631         # just the media without qualities renditions.
1632         # Fortunately, master playlist can be easily distinguished from media
1633         # playlist based on particular tags availability. As of [1, 4.3.3, 4.3.4]
1634         # master playlist tags MUST NOT appear in a media playist and vice versa.
1635         # As of [1, 4.3.3.1] #EXT-X-TARGETDURATION tag is REQUIRED for every
1636         # media playlist and MUST NOT appear in master playlist thus we can
1637         # clearly detect media playlist with this criterion.
1638
1639         if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
1640             return [{
1641                 'url': m3u8_url,
1642                 'format_id': m3u8_id,
1643                 'ext': ext,
1644                 'protocol': entry_protocol,
1645                 'preference': preference,
1646             }]
1647
1648         groups = {}
1649         last_stream_inf = {}
1650
1651         def extract_media(x_media_line):
1652             media = parse_m3u8_attributes(x_media_line)
1653             # As per [1, 4.3.4.1] TYPE, GROUP-ID and NAME are REQUIRED
1654             media_type, group_id, name = media.get('TYPE'), media.get('GROUP-ID'), media.get('NAME')
1655             if not (media_type and group_id and name):
1656                 return
1657             groups.setdefault(group_id, []).append(media)
1658             if media_type not in ('VIDEO', 'AUDIO'):
1659                 return
1660             media_url = media.get('URI')
1661             if media_url:
1662                 format_id = []
1663                 for v in (m3u8_id, group_id, name):
1664                     if v:
1665                         format_id.append(v)
1666                 f = {
1667                     'format_id': '-'.join(format_id),
1668                     'url': format_url(media_url),
1669                     'manifest_url': m3u8_url,
1670                     'language': media.get('LANGUAGE'),
1671                     'ext': ext,
1672                     'protocol': entry_protocol,
1673                     'preference': preference,
1674                 }
1675                 if media_type == 'AUDIO':
1676                     f['vcodec'] = 'none'
1677                 formats.append(f)
1678
1679         def build_stream_name():
1680             # Despite specification does not mention NAME attribute for
1681             # EXT-X-STREAM-INF tag it still sometimes may be present (see [1]
1682             # or vidio test in TestInfoExtractor.test_parse_m3u8_formats)
1683             # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
1684             stream_name = last_stream_inf.get('NAME')
1685             if stream_name:
1686                 return stream_name
1687             # If there is no NAME in EXT-X-STREAM-INF it will be obtained
1688             # from corresponding rendition group
1689             stream_group_id = last_stream_inf.get('VIDEO')
1690             if not stream_group_id:
1691                 return
1692             stream_group = groups.get(stream_group_id)
1693             if not stream_group:
1694                 return stream_group_id
1695             rendition = stream_group[0]
1696             return rendition.get('NAME') or stream_group_id
1697
1698         # parse EXT-X-MEDIA tags before EXT-X-STREAM-INF in order to have the
1699         # chance to detect video only formats when EXT-X-STREAM-INF tags
1700         # precede EXT-X-MEDIA tags in HLS manifest such as [3].
1701         for line in m3u8_doc.splitlines():
1702             if line.startswith('#EXT-X-MEDIA:'):
1703                 extract_media(line)
1704
1705         for line in m3u8_doc.splitlines():
1706             if line.startswith('#EXT-X-STREAM-INF:'):
1707                 last_stream_inf = parse_m3u8_attributes(line)
1708             elif line.startswith('#') or not line.strip():
1709                 continue
1710             else:
1711                 tbr = float_or_none(
1712                     last_stream_inf.get('AVERAGE-BANDWIDTH')
1713                     or last_stream_inf.get('BANDWIDTH'), scale=1000)
1714                 format_id = []
1715                 if m3u8_id:
1716                     format_id.append(m3u8_id)
1717                 stream_name = build_stream_name()
1718                 # Bandwidth of live streams may differ over time thus making
1719                 # format_id unpredictable. So it's better to keep provided
1720                 # format_id intact.
1721                 if not live:
1722                     format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
1723                 manifest_url = format_url(line.strip())
1724                 f = {
1725                     'format_id': '-'.join(format_id),
1726                     'url': manifest_url,
1727                     'manifest_url': m3u8_url,
1728                     'tbr': tbr,
1729                     'ext': ext,
1730                     'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
1731                     'protocol': entry_protocol,
1732                     'preference': preference,
1733                 }
1734                 resolution = last_stream_inf.get('RESOLUTION')
1735                 if resolution:
1736                     mobj = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', resolution)
1737                     if mobj:
1738                         f['width'] = int(mobj.group('width'))
1739                         f['height'] = int(mobj.group('height'))
1740                 # Unified Streaming Platform
1741                 mobj = re.search(
1742                     r'audio.*?(?:%3D|=)(\d+)(?:-video.*?(?:%3D|=)(\d+))?', f['url'])
1743                 if mobj:
1744                     abr, vbr = mobj.groups()
1745                     abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
1746                     f.update({
1747                         'vbr': vbr,
1748                         'abr': abr,
1749                     })
1750                 codecs = parse_codecs(last_stream_inf.get('CODECS'))
1751                 f.update(codecs)
1752                 audio_group_id = last_stream_inf.get('AUDIO')
1753                 # As per [1, 4.3.4.1.1] any EXT-X-STREAM-INF tag which
1754                 # references a rendition group MUST have a CODECS attribute.
1755                 # However, this is not always respected, for example, [2]
1756                 # contains EXT-X-STREAM-INF tag which references AUDIO
1757                 # rendition group but does not have CODECS and despite
1758                 # referencing an audio group it represents a complete
1759                 # (with audio and video) format. So, for such cases we will
1760                 # ignore references to rendition groups and treat them
1761                 # as complete formats.
1762                 if audio_group_id and codecs and f.get('vcodec') != 'none':
1763                     audio_group = groups.get(audio_group_id)
1764                     if audio_group and audio_group[0].get('URI'):
1765                         # TODO: update acodec for audio only formats with
1766                         # the same GROUP-ID
1767                         f['acodec'] = 'none'
1768                 formats.append(f)
1769
1770                 # for DailyMotion
1771                 progressive_uri = last_stream_inf.get('PROGRESSIVE-URI')
1772                 if progressive_uri:
1773                     http_f = f.copy()
1774                     del http_f['manifest_url']
1775                     http_f.update({
1776                         'format_id': f['format_id'].replace('hls-', 'http-'),
1777                         'protocol': 'http',
1778                         'url': progressive_uri,
1779                     })
1780                     formats.append(http_f)
1781
1782                 last_stream_inf = {}
1783         return formats
1784
1785     @staticmethod
1786     def _xpath_ns(path, namespace=None):
1787         if not namespace:
1788             return path
1789         out = []
1790         for c in path.split('/'):
1791             if not c or c == '.':
1792                 out.append(c)
1793             else:
1794                 out.append('{%s}%s' % (namespace, c))
1795         return '/'.join(out)
1796
1797     def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
1798         smil = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
1799
1800         if smil is False:
1801             assert not fatal
1802             return []
1803
1804         namespace = self._parse_smil_namespace(smil)
1805
1806         return self._parse_smil_formats(
1807             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1808
1809     def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
1810         smil = self._download_smil(smil_url, video_id, fatal=fatal)
1811         if smil is False:
1812             return {}
1813         return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
1814
1815     def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
1816         return self._download_xml(
1817             smil_url, video_id, 'Downloading SMIL file',
1818             'Unable to download SMIL file', fatal=fatal, transform_source=transform_source)
1819
1820     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
1821         namespace = self._parse_smil_namespace(smil)
1822
1823         formats = self._parse_smil_formats(
1824             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1825         subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
1826
1827         video_id = os.path.splitext(url_basename(smil_url))[0]
1828         title = None
1829         description = None
1830         upload_date = None
1831         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1832             name = meta.attrib.get('name')
1833             content = meta.attrib.get('content')
1834             if not name or not content:
1835                 continue
1836             if not title and name == 'title':
1837                 title = content
1838             elif not description and name in ('description', 'abstract'):
1839                 description = content
1840             elif not upload_date and name == 'date':
1841                 upload_date = unified_strdate(content)
1842
1843         thumbnails = [{
1844             'id': image.get('type'),
1845             'url': image.get('src'),
1846             'width': int_or_none(image.get('width')),
1847             'height': int_or_none(image.get('height')),
1848         } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
1849
1850         return {
1851             'id': video_id,
1852             'title': title or video_id,
1853             'description': description,
1854             'upload_date': upload_date,
1855             'thumbnails': thumbnails,
1856             'formats': formats,
1857             'subtitles': subtitles,
1858         }
1859
1860     def _parse_smil_namespace(self, smil):
1861         return self._search_regex(
1862             r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
1863
1864     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
1865         base = smil_url
1866         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1867             b = meta.get('base') or meta.get('httpBase')
1868             if b:
1869                 base = b
1870                 break
1871
1872         formats = []
1873         rtmp_count = 0
1874         http_count = 0
1875         m3u8_count = 0
1876
1877         srcs = []
1878         media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
1879         for medium in media:
1880             src = medium.get('src')
1881             if not src or src in srcs:
1882                 continue
1883             srcs.append(src)
1884
1885             bitrate = float_or_none(medium.get('system-bitrate') or medium.get('systemBitrate'), 1000)
1886             filesize = int_or_none(medium.get('size') or medium.get('fileSize'))
1887             width = int_or_none(medium.get('width'))
1888             height = int_or_none(medium.get('height'))
1889             proto = medium.get('proto')
1890             ext = medium.get('ext')
1891             src_ext = determine_ext(src)
1892             streamer = medium.get('streamer') or base
1893
1894             if proto == 'rtmp' or streamer.startswith('rtmp'):
1895                 rtmp_count += 1
1896                 formats.append({
1897                     'url': streamer,
1898                     'play_path': src,
1899                     'ext': 'flv',
1900                     'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
1901                     'tbr': bitrate,
1902                     'filesize': filesize,
1903                     'width': width,
1904                     'height': height,
1905                 })
1906                 if transform_rtmp_url:
1907                     streamer, src = transform_rtmp_url(streamer, src)
1908                     formats[-1].update({
1909                         'url': streamer,
1910                         'play_path': src,
1911                     })
1912                 continue
1913
1914             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
1915             src_url = src_url.strip()
1916
1917             if proto == 'm3u8' or src_ext == 'm3u8':
1918                 m3u8_formats = self._extract_m3u8_formats(
1919                     src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
1920                 if len(m3u8_formats) == 1:
1921                     m3u8_count += 1
1922                     m3u8_formats[0].update({
1923                         'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
1924                         'tbr': bitrate,
1925                         'width': width,
1926                         'height': height,
1927                     })
1928                 formats.extend(m3u8_formats)
1929             elif src_ext == 'f4m':
1930                 f4m_url = src_url
1931                 if not f4m_params:
1932                     f4m_params = {
1933                         'hdcore': '3.2.0',
1934                         'plugin': 'flowplayer-3.2.0.1',
1935                     }
1936                 f4m_url += '&' if '?' in f4m_url else '?'
1937                 f4m_url += compat_urllib_parse_urlencode(f4m_params)
1938                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
1939             elif src_ext == 'mpd':
1940                 formats.extend(self._extract_mpd_formats(
1941                     src_url, video_id, mpd_id='dash', fatal=False))
1942             elif re.search(r'\.ism/[Mm]anifest', src_url):
1943                 formats.extend(self._extract_ism_formats(
1944                     src_url, video_id, ism_id='mss', fatal=False))
1945             elif src_url.startswith('http') and self._is_valid_url(src, video_id):
1946                 http_count += 1
1947                 formats.append({
1948                     'url': src_url,
1949                     'ext': ext or src_ext or 'flv',
1950                     'format_id': 'http-%d' % (bitrate or http_count),
1951                     'tbr': bitrate,
1952                     'filesize': filesize,
1953                     'width': width,
1954                     'height': height,
1955                 })
1956
1957         return formats
1958
1959     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
1960         urls = []
1961         subtitles = {}
1962         for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
1963             src = textstream.get('src')
1964             if not src or src in urls:
1965                 continue
1966             urls.append(src)
1967             ext = textstream.get('ext') or mimetype2ext(textstream.get('type')) or determine_ext(src)
1968             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
1969             subtitles.setdefault(lang, []).append({
1970                 'url': src,
1971                 'ext': ext,
1972             })
1973         return subtitles
1974
1975     def _extract_xspf_playlist(self, xspf_url, playlist_id, fatal=True):
1976         xspf = self._download_xml(
1977             xspf_url, playlist_id, 'Downloading xpsf playlist',
1978             'Unable to download xspf manifest', fatal=fatal)
1979         if xspf is False:
1980             return []
1981         return self._parse_xspf(
1982             xspf, playlist_id, xspf_url=xspf_url,
1983             xspf_base_url=base_url(xspf_url))
1984
1985     def _parse_xspf(self, xspf_doc, playlist_id, xspf_url=None, xspf_base_url=None):
1986         NS_MAP = {
1987             'xspf': 'http://xspf.org/ns/0/',
1988             's1': 'http://static.streamone.nl/player/ns/0',
1989         }
1990
1991         entries = []
1992         for track in xspf_doc.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
1993             title = xpath_text(
1994                 track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
1995             description = xpath_text(
1996                 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
1997             thumbnail = xpath_text(
1998                 track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
1999             duration = float_or_none(
2000                 xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
2001
2002             formats = []
2003             for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP)):
2004                 format_url = urljoin(xspf_base_url, location.text)
2005                 if not format_url:
2006                     continue
2007                 formats.append({
2008                     'url': format_url,
2009                     'manifest_url': xspf_url,
2010                     'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
2011                     'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
2012                     'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
2013                 })
2014             self._sort_formats(formats)
2015
2016             entries.append({
2017                 'id': playlist_id,
2018                 'title': title,
2019                 'description': description,
2020                 'thumbnail': thumbnail,
2021                 'duration': duration,
2022                 'formats': formats,
2023             })
2024         return entries
2025
2026     def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}, data=None, headers={}, query={}):
2027         res = self._download_xml_handle(
2028             mpd_url, video_id,
2029             note=note or 'Downloading MPD manifest',
2030             errnote=errnote or 'Failed to download MPD manifest',
2031             fatal=fatal, data=data, headers=headers, query=query)
2032         if res is False:
2033             return []
2034         mpd_doc, urlh = res
2035         if mpd_doc is None:
2036             return []
2037         mpd_base_url = base_url(urlh.geturl())
2038
2039         return self._parse_mpd_formats(
2040             mpd_doc, mpd_id=mpd_id, mpd_base_url=mpd_base_url,
2041             formats_dict=formats_dict, mpd_url=mpd_url)
2042
2043     def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}, mpd_url=None):
2044         """
2045         Parse formats from MPD manifest.
2046         References:
2047          1. MPEG-DASH Standard, ISO/IEC 23009-1:2014(E),
2048             http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip
2049          2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
2050         """
2051         if mpd_doc.get('type') == 'dynamic':
2052             return []
2053
2054         namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
2055
2056         def _add_ns(path):
2057             return self._xpath_ns(path, namespace)
2058
2059         def is_drm_protected(element):
2060             return element.find(_add_ns('ContentProtection')) is not None
2061
2062         def extract_multisegment_info(element, ms_parent_info):
2063             ms_info = ms_parent_info.copy()
2064
2065             # As per [1, 5.3.9.2.2] SegmentList and SegmentTemplate share some
2066             # common attributes and elements.  We will only extract relevant
2067             # for us.
2068             def extract_common(source):
2069                 segment_timeline = source.find(_add_ns('SegmentTimeline'))
2070                 if segment_timeline is not None:
2071                     s_e = segment_timeline.findall(_add_ns('S'))
2072                     if s_e:
2073                         ms_info['total_number'] = 0
2074                         ms_info['s'] = []
2075                         for s in s_e:
2076                             r = int(s.get('r', 0))
2077                             ms_info['total_number'] += 1 + r
2078                             ms_info['s'].append({
2079                                 't': int(s.get('t', 0)),
2080                                 # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
2081                                 'd': int(s.attrib['d']),
2082                                 'r': r,
2083                             })
2084                 start_number = source.get('startNumber')
2085                 if start_number:
2086                     ms_info['start_number'] = int(start_number)
2087                 timescale = source.get('timescale')
2088                 if timescale:
2089                     ms_info['timescale'] = int(timescale)
2090                 segment_duration = source.get('duration')
2091                 if segment_duration:
2092                     ms_info['segment_duration'] = float(segment_duration)
2093
2094             def extract_Initialization(source):
2095                 initialization = source.find(_add_ns('Initialization'))
2096                 if initialization is not None:
2097                     ms_info['initialization_url'] = initialization.attrib['sourceURL']
2098
2099             segment_list = element.find(_add_ns('SegmentList'))
2100             if segment_list is not None:
2101                 extract_common(segment_list)
2102                 extract_Initialization(segment_list)
2103                 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
2104                 if segment_urls_e:
2105                     ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
2106             else:
2107                 segment_template = element.find(_add_ns('SegmentTemplate'))
2108                 if segment_template is not None:
2109                     extract_common(segment_template)
2110                     media = segment_template.get('media')
2111                     if media:
2112                         ms_info['media'] = media
2113                     initialization = segment_template.get('initialization')
2114                     if initialization:
2115                         ms_info['initialization'] = initialization
2116                     else:
2117                         extract_Initialization(segment_template)
2118             return ms_info
2119
2120         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
2121         formats = []
2122         for period in mpd_doc.findall(_add_ns('Period')):
2123             period_duration = parse_duration(period.get('duration')) or mpd_duration
2124             period_ms_info = extract_multisegment_info(period, {
2125                 'start_number': 1,
2126                 'timescale': 1,
2127             })
2128             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
2129                 if is_drm_protected(adaptation_set):
2130                     continue
2131                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
2132                 for representation in adaptation_set.findall(_add_ns('Representation')):
2133                     if is_drm_protected(representation):
2134                         continue
2135                     representation_attrib = adaptation_set.attrib.copy()
2136                     representation_attrib.update(representation.attrib)
2137                     # According to [1, 5.3.7.2, Table 9, page 41], @mimeType is mandatory
2138                     mime_type = representation_attrib['mimeType']
2139                     content_type = mime_type.split('/')[0]
2140                     if content_type == 'text':
2141                         # TODO implement WebVTT downloading
2142                         pass
2143                     elif content_type in ('video', 'audio'):
2144                         base_url = ''
2145                         for element in (representation, adaptation_set, period, mpd_doc):
2146                             base_url_e = element.find(_add_ns('BaseURL'))
2147                             if base_url_e is not None:
2148                                 base_url = base_url_e.text + base_url
2149                                 if re.match(r'^https?://', base_url):
2150                                     break
2151                         if mpd_base_url and not re.match(r'^https?://', base_url):
2152                             if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
2153                                 mpd_base_url += '/'
2154                             base_url = mpd_base_url + base_url
2155                         representation_id = representation_attrib.get('id')
2156                         lang = representation_attrib.get('lang')
2157                         url_el = representation.find(_add_ns('BaseURL'))
2158                         filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
2159                         bandwidth = int_or_none(representation_attrib.get('bandwidth'))
2160                         f = {
2161                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
2162                             'manifest_url': mpd_url,
2163                             'ext': mimetype2ext(mime_type),
2164                             'width': int_or_none(representation_attrib.get('width')),
2165                             'height': int_or_none(representation_attrib.get('height')),
2166                             'tbr': float_or_none(bandwidth, 1000),
2167                             'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
2168                             'fps': int_or_none(representation_attrib.get('frameRate')),
2169                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
2170                             'format_note': 'DASH %s' % content_type,
2171                             'filesize': filesize,
2172                             'container': mimetype2ext(mime_type) + '_dash',
2173                         }
2174                         f.update(parse_codecs(representation_attrib.get('codecs')))
2175                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
2176
2177                         def prepare_template(template_name, identifiers):
2178                             tmpl = representation_ms_info[template_name]
2179                             # First of, % characters outside $...$ templates
2180                             # must be escaped by doubling for proper processing
2181                             # by % operator string formatting used further (see
2182                             # https://github.com/ytdl-org/youtube-dl/issues/16867).
2183                             t = ''
2184                             in_template = False
2185                             for c in tmpl:
2186                                 t += c
2187                                 if c == '$':
2188                                     in_template = not in_template
2189                                 elif c == '%' and not in_template:
2190                                     t += c
2191                             # Next, $...$ templates are translated to their
2192                             # %(...) counterparts to be used with % operator
2193                             t = t.replace('$RepresentationID$', representation_id)
2194                             t = re.sub(r'\$(%s)\$' % '|'.join(identifiers), r'%(\1)d', t)
2195                             t = re.sub(r'\$(%s)%%([^$]+)\$' % '|'.join(identifiers), r'%(\1)\2', t)
2196                             t.replace('$$', '$')
2197                             return t
2198
2199                         # @initialization is a regular template like @media one
2200                         # so it should be handled just the same way (see
2201                         # https://github.com/ytdl-org/youtube-dl/issues/11605)
2202                         if 'initialization' in representation_ms_info:
2203                             initialization_template = prepare_template(
2204                                 'initialization',
2205                                 # As per [1, 5.3.9.4.2, Table 15, page 54] $Number$ and
2206                                 # $Time$ shall not be included for @initialization thus
2207                                 # only $Bandwidth$ remains
2208                                 ('Bandwidth', ))
2209                             representation_ms_info['initialization_url'] = initialization_template % {
2210                                 'Bandwidth': bandwidth,
2211                             }
2212
2213                         def location_key(location):
2214                             return 'url' if re.match(r'^https?://', location) else 'path'
2215
2216                         if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
2217
2218                             media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
2219                             media_location_key = location_key(media_template)
2220
2221                             # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
2222                             # can't be used at the same time
2223                             if '%(Number' in media_template and 's' not in representation_ms_info:
2224                                 segment_duration = None
2225                                 if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
2226                                     segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
2227                                     representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
2228                                 representation_ms_info['fragments'] = [{
2229                                     media_location_key: media_template % {
2230                                         'Number': segment_number,
2231                                         'Bandwidth': bandwidth,
2232                                     },
2233                                     'duration': segment_duration,
2234                                 } for segment_number in range(
2235                                     representation_ms_info['start_number'],
2236                                     representation_ms_info['total_number'] + representation_ms_info['start_number'])]
2237                             else:
2238                                 # $Number*$ or $Time$ in media template with S list available
2239                                 # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
2240                                 # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
2241                                 representation_ms_info['fragments'] = []
2242                                 segment_time = 0
2243                                 segment_d = None
2244                                 segment_number = representation_ms_info['start_number']
2245
2246                                 def add_segment_url():
2247                                     segment_url = media_template % {
2248                                         'Time': segment_time,
2249                                         'Bandwidth': bandwidth,
2250                                         'Number': segment_number,
2251                                     }
2252                                     representation_ms_info['fragments'].append({
2253                                         media_location_key: segment_url,
2254                                         'duration': float_or_none(segment_d, representation_ms_info['timescale']),
2255                                     })
2256
2257                                 for num, s in enumerate(representation_ms_info['s']):
2258                                     segment_time = s.get('t') or segment_time
2259                                     segment_d = s['d']
2260                                     add_segment_url()
2261                                     segment_number += 1
2262                                     for r in range(s.get('r', 0)):
2263                                         segment_time += segment_d
2264                                         add_segment_url()
2265                                         segment_number += 1
2266                                     segment_time += segment_d
2267                         elif 'segment_urls' in representation_ms_info and 's' in representation_ms_info:
2268                             # No media template
2269                             # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
2270                             # or any YouTube dashsegments video
2271                             fragments = []
2272                             segment_index = 0
2273                             timescale = representation_ms_info['timescale']
2274                             for s in representation_ms_info['s']:
2275                                 duration = float_or_none(s['d'], timescale)
2276                                 for r in range(s.get('r', 0) + 1):
2277                                     segment_uri = representation_ms_info['segment_urls'][segment_index]
2278                                     fragments.append({
2279                                         location_key(segment_uri): segment_uri,
2280                                         'duration': duration,
2281                                     })
2282                                     segment_index += 1
2283                             representation_ms_info['fragments'] = fragments
2284                         elif 'segment_urls' in representation_ms_info:
2285                             # Segment URLs with no SegmentTimeline
2286                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
2287                             # https://github.com/ytdl-org/youtube-dl/pull/14844
2288                             fragments = []
2289                             segment_duration = float_or_none(
2290                                 representation_ms_info['segment_duration'],
2291                                 representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
2292                             for segment_url in representation_ms_info['segment_urls']:
2293                                 fragment = {
2294                                     location_key(segment_url): segment_url,
2295                                 }
2296                                 if segment_duration:
2297                                     fragment['duration'] = segment_duration
2298                                 fragments.append(fragment)
2299                             representation_ms_info['fragments'] = fragments
2300                         # If there is a fragments key available then we correctly recognized fragmented media.
2301                         # Otherwise we will assume unfragmented media with direct access. Technically, such
2302                         # assumption is not necessarily correct since we may simply have no support for
2303                         # some forms of fragmented media renditions yet, but for now we'll use this fallback.
2304                         if 'fragments' in representation_ms_info:
2305                             f.update({
2306                                 # NB: mpd_url may be empty when MPD manifest is parsed from a string
2307                                 'url': mpd_url or base_url,
2308                                 'fragment_base_url': base_url,
2309                                 'fragments': [],
2310                                 'protocol': 'http_dash_segments',
2311                             })
2312                             if 'initialization_url' in representation_ms_info:
2313                                 initialization_url = representation_ms_info['initialization_url']
2314                                 if not f.get('url'):
2315                                     f['url'] = initialization_url
2316                                 f['fragments'].append({location_key(initialization_url): initialization_url})
2317                             f['fragments'].extend(representation_ms_info['fragments'])
2318                         else:
2319                             # Assuming direct URL to unfragmented media.
2320                             f['url'] = base_url
2321
2322                         # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
2323                         # is not necessarily unique within a Period thus formats with
2324                         # the same `format_id` are quite possible. There are numerous examples
2325                         # of such manifests (see https://github.com/ytdl-org/youtube-dl/issues/15111,
2326                         # https://github.com/ytdl-org/youtube-dl/issues/13919)
2327                         full_info = formats_dict.get(representation_id, {}).copy()
2328                         full_info.update(f)
2329                         formats.append(full_info)
2330                     else:
2331                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
2332         return formats
2333
2334     def _extract_ism_formats(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
2335         res = self._download_xml_handle(
2336             ism_url, video_id,
2337             note=note or 'Downloading ISM manifest',
2338             errnote=errnote or 'Failed to download ISM manifest',
2339             fatal=fatal, data=data, headers=headers, query=query)
2340         if res is False:
2341             return []
2342         ism_doc, urlh = res
2343
2344         return self._parse_ism_formats(ism_doc, urlh.geturl(), ism_id)
2345
2346     def _parse_ism_formats(self, ism_doc, ism_url, ism_id=None):
2347         """
2348         Parse formats from ISM manifest.
2349         References:
2350          1. [MS-SSTR]: Smooth Streaming Protocol,
2351             https://msdn.microsoft.com/en-us/library/ff469518.aspx
2352         """
2353         if ism_doc.get('IsLive') == 'TRUE' or ism_doc.find('Protection') is not None:
2354             return []
2355
2356         duration = int(ism_doc.attrib['Duration'])
2357         timescale = int_or_none(ism_doc.get('TimeScale')) or 10000000
2358
2359         formats = []
2360         for stream in ism_doc.findall('StreamIndex'):
2361             stream_type = stream.get('Type')
2362             if stream_type not in ('video', 'audio'):
2363                 continue
2364             url_pattern = stream.attrib['Url']
2365             stream_timescale = int_or_none(stream.get('TimeScale')) or timescale
2366             stream_name = stream.get('Name')
2367             for track in stream.findall('QualityLevel'):
2368                 fourcc = track.get('FourCC', 'AACL' if track.get('AudioTag') == '255' else None)
2369                 # TODO: add support for WVC1 and WMAP
2370                 if fourcc not in ('H264', 'AVC1', 'AACL'):
2371                     self.report_warning('%s is not a supported codec' % fourcc)
2372                     continue
2373                 tbr = int(track.attrib['Bitrate']) // 1000
2374                 # [1] does not mention Width and Height attributes. However,
2375                 # they're often present while MaxWidth and MaxHeight are
2376                 # missing, so should be used as fallbacks
2377                 width = int_or_none(track.get('MaxWidth') or track.get('Width'))
2378                 height = int_or_none(track.get('MaxHeight') or track.get('Height'))
2379                 sampling_rate = int_or_none(track.get('SamplingRate'))
2380
2381                 track_url_pattern = re.sub(r'{[Bb]itrate}', track.attrib['Bitrate'], url_pattern)
2382                 track_url_pattern = compat_urlparse.urljoin(ism_url, track_url_pattern)
2383
2384                 fragments = []
2385                 fragment_ctx = {
2386                     'time': 0,
2387                 }
2388                 stream_fragments = stream.findall('c')
2389                 for stream_fragment_index, stream_fragment in enumerate(stream_fragments):
2390                     fragment_ctx['time'] = int_or_none(stream_fragment.get('t')) or fragment_ctx['time']
2391                     fragment_repeat = int_or_none(stream_fragment.get('r')) or 1
2392                     fragment_ctx['duration'] = int_or_none(stream_fragment.get('d'))
2393                     if not fragment_ctx['duration']:
2394                         try:
2395                             next_fragment_time = int(stream_fragment[stream_fragment_index + 1].attrib['t'])
2396                         except IndexError:
2397                             next_fragment_time = duration
2398                         fragment_ctx['duration'] = (next_fragment_time - fragment_ctx['time']) / fragment_repeat
2399                     for _ in range(fragment_repeat):
2400                         fragments.append({
2401                             'url': re.sub(r'{start[ _]time}', compat_str(fragment_ctx['time']), track_url_pattern),
2402                             'duration': fragment_ctx['duration'] / stream_timescale,
2403                         })
2404                         fragment_ctx['time'] += fragment_ctx['duration']
2405
2406                 format_id = []
2407                 if ism_id:
2408                     format_id.append(ism_id)
2409                 if stream_name:
2410                     format_id.append(stream_name)
2411                 format_id.append(compat_str(tbr))
2412
2413                 formats.append({
2414                     'format_id': '-'.join(format_id),
2415                     'url': ism_url,
2416                     'manifest_url': ism_url,
2417                     'ext': 'ismv' if stream_type == 'video' else 'isma',
2418                     'width': width,
2419                     'height': height,
2420                     'tbr': tbr,
2421                     'asr': sampling_rate,
2422                     'vcodec': 'none' if stream_type == 'audio' else fourcc,
2423                     'acodec': 'none' if stream_type == 'video' else fourcc,
2424                     'protocol': 'ism',
2425                     'fragments': fragments,
2426                     '_download_params': {
2427                         'duration': duration,
2428                         'timescale': stream_timescale,
2429                         'width': width or 0,
2430                         'height': height or 0,
2431                         'fourcc': fourcc,
2432                         'codec_private_data': track.get('CodecPrivateData'),
2433                         'sampling_rate': sampling_rate,
2434                         'channels': int_or_none(track.get('Channels', 2)),
2435                         'bits_per_sample': int_or_none(track.get('BitsPerSample', 16)),
2436                         'nal_unit_length_field': int_or_none(track.get('NALUnitLengthField', 4)),
2437                     },
2438                 })
2439         return formats
2440
2441     def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8', mpd_id=None, preference=None):
2442         def absolute_url(item_url):
2443             return urljoin(base_url, item_url)
2444
2445         def parse_content_type(content_type):
2446             if not content_type:
2447                 return {}
2448             ctr = re.search(r'(?P<mimetype>[^/]+/[^;]+)(?:;\s*codecs="?(?P<codecs>[^"]+))?', content_type)
2449             if ctr:
2450                 mimetype, codecs = ctr.groups()
2451                 f = parse_codecs(codecs)
2452                 f['ext'] = mimetype2ext(mimetype)
2453                 return f
2454             return {}
2455
2456         def _media_formats(src, cur_media_type, type_info={}):
2457             full_url = absolute_url(src)
2458             ext = type_info.get('ext') or determine_ext(full_url)
2459             if ext == 'm3u8':
2460                 is_plain_url = False
2461                 formats = self._extract_m3u8_formats(
2462                     full_url, video_id, ext='mp4',
2463                     entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id,
2464                     preference=preference, fatal=False)
2465             elif ext == 'mpd':
2466                 is_plain_url = False
2467                 formats = self._extract_mpd_formats(
2468                     full_url, video_id, mpd_id=mpd_id, fatal=False)
2469             else:
2470                 is_plain_url = True
2471                 formats = [{
2472                     'url': full_url,
2473                     'vcodec': 'none' if cur_media_type == 'audio' else None,
2474                 }]
2475             return is_plain_url, formats
2476
2477         entries = []
2478         # amp-video and amp-audio are very similar to their HTML5 counterparts
2479         # so we wll include them right here (see
2480         # https://www.ampproject.org/docs/reference/components/amp-video)
2481         media_tags = [(media_tag, media_type, '')
2482                       for media_tag, media_type
2483                       in re.findall(r'(?s)(<(?:amp-)?(video|audio)[^>]*/>)', webpage)]
2484         media_tags.extend(re.findall(
2485             # We only allow video|audio followed by a whitespace or '>'.
2486             # Allowing more characters may end up in significant slow down (see
2487             # https://github.com/ytdl-org/youtube-dl/issues/11979, example URL:
2488             # http://www.porntrex.com/maps/videositemap.xml).
2489             r'(?s)(<(?P<tag>(?:amp-)?(?:video|audio))(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
2490         for media_tag, media_type, media_content in media_tags:
2491             media_info = {
2492                 'formats': [],
2493                 'subtitles': {},
2494             }
2495             media_attributes = extract_attributes(media_tag)
2496             src = strip_or_none(media_attributes.get('src'))
2497             if src:
2498                 _, formats = _media_formats(src, media_type)
2499                 media_info['formats'].extend(formats)
2500             media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
2501             if media_content:
2502                 for source_tag in re.findall(r'<source[^>]+>', media_content):
2503                     s_attr = extract_attributes(source_tag)
2504                     # data-video-src and data-src are non standard but seen
2505                     # several times in the wild
2506                     src = strip_or_none(dict_get(s_attr, ('src', 'data-video-src', 'data-src')))
2507                     if not src:
2508                         continue
2509                     f = parse_content_type(s_attr.get('type'))
2510                     is_plain_url, formats = _media_formats(src, media_type, f)
2511                     if is_plain_url:
2512                         # width, height, res, label and title attributes are
2513                         # all not standard but seen several times in the wild
2514                         labels = [
2515                             s_attr.get(lbl)
2516                             for lbl in ('label', 'title')
2517                             if str_or_none(s_attr.get(lbl))
2518                         ]
2519                         width = int_or_none(s_attr.get('width'))
2520                         height = (int_or_none(s_attr.get('height'))
2521                                   or int_or_none(s_attr.get('res')))
2522                         if not width or not height:
2523                             for lbl in labels:
2524                                 resolution = parse_resolution(lbl)
2525                                 if not resolution:
2526                                     continue
2527                                 width = width or resolution.get('width')
2528                                 height = height or resolution.get('height')
2529                         for lbl in labels:
2530                             tbr = parse_bitrate(lbl)
2531                             if tbr:
2532                                 break
2533                         else:
2534                             tbr = None
2535                         f.update({
2536                             'width': width,
2537                             'height': height,
2538                             'tbr': tbr,
2539                             'format_id': s_attr.get('label') or s_attr.get('title'),
2540                         })
2541                         f.update(formats[0])
2542                         media_info['formats'].append(f)
2543                     else:
2544                         media_info['formats'].extend(formats)
2545                 for track_tag in re.findall(r'<track[^>]+>', media_content):
2546                     track_attributes = extract_attributes(track_tag)
2547                     kind = track_attributes.get('kind')
2548                     if not kind or kind in ('subtitles', 'captions'):
2549                         src = strip_or_none(track_attributes.get('src'))
2550                         if not src:
2551                             continue
2552                         lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')
2553                         media_info['subtitles'].setdefault(lang, []).append({
2554                             'url': absolute_url(src),
2555                         })
2556             for f in media_info['formats']:
2557                 f.setdefault('http_headers', {})['Referer'] = base_url
2558             if media_info['formats'] or media_info['subtitles']:
2559                 entries.append(media_info)
2560         return entries
2561
2562     def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
2563         formats = []
2564         hdcore_sign = 'hdcore=3.7.0'
2565         f4m_url = re.sub(r'(https?://[^/]+)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
2566         hds_host = hosts.get('hds')
2567         if hds_host:
2568             f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)
2569         if 'hdcore=' not in f4m_url:
2570             f4m_url += ('&' if '?' in f4m_url else '?') + hdcore_sign
2571         f4m_formats = self._extract_f4m_formats(
2572             f4m_url, video_id, f4m_id='hds', fatal=False)
2573         for entry in f4m_formats:
2574             entry.update({'extra_param_to_segment_url': hdcore_sign})
2575         formats.extend(f4m_formats)
2576         m3u8_url = re.sub(r'(https?://[^/]+)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
2577         hls_host = hosts.get('hls')
2578         if hls_host:
2579             m3u8_url = re.sub(r'(https?://)[^/]+', r'\1' + hls_host, m3u8_url)
2580         formats.extend(self._extract_m3u8_formats(
2581             m3u8_url, video_id, 'mp4', 'm3u8_native',
2582             m3u8_id='hls', fatal=False))
2583         return formats
2584
2585     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
2586         query = compat_urlparse.urlparse(url).query
2587         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
2588         mobj = re.search(
2589             r'(?:(?:http|rtmp|rtsp)(?P<s>s)?:)?(?P<url>//[^?]+)', url)
2590         url_base = mobj.group('url')
2591         http_base_url = '%s%s:%s' % ('http', mobj.group('s') or '', url_base)
2592         formats = []
2593
2594         def manifest_url(manifest):
2595             m_url = '%s/%s' % (http_base_url, manifest)
2596             if query:
2597                 m_url += '?%s' % query
2598             return m_url
2599
2600         if 'm3u8' not in skip_protocols:
2601             formats.extend(self._extract_m3u8_formats(
2602                 manifest_url('playlist.m3u8'), video_id, 'mp4',
2603                 m3u8_entry_protocol, m3u8_id='hls', fatal=False))
2604         if 'f4m' not in skip_protocols:
2605             formats.extend(self._extract_f4m_formats(
2606                 manifest_url('manifest.f4m'),
2607                 video_id, f4m_id='hds', fatal=False))
2608         if 'dash' not in skip_protocols:
2609             formats.extend(self._extract_mpd_formats(
2610                 manifest_url('manifest.mpd'),
2611                 video_id, mpd_id='dash', fatal=False))
2612         if re.search(r'(?:/smil:|\.smil)', url_base):
2613             if 'smil' not in skip_protocols:
2614                 rtmp_formats = self._extract_smil_formats(
2615                     manifest_url('jwplayer.smil'),
2616                     video_id, fatal=False)
2617                 for rtmp_format in rtmp_formats:
2618                     rtsp_format = rtmp_format.copy()
2619                     rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
2620                     del rtsp_format['play_path']
2621                     del rtsp_format['ext']
2622                     rtsp_format.update({
2623                         'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
2624                         'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
2625                         'protocol': 'rtsp',
2626                     })
2627                     formats.extend([rtmp_format, rtsp_format])
2628         else:
2629             for protocol in ('rtmp', 'rtsp'):
2630                 if protocol not in skip_protocols:
2631                     formats.append({
2632                         'url': '%s:%s' % (protocol, url_base),
2633                         'format_id': protocol,
2634                         'protocol': protocol,
2635                     })
2636         return formats
2637
2638     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
2639         mobj = re.search(
2640             r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
2641             webpage)
2642         if mobj:
2643             try:
2644                 jwplayer_data = self._parse_json(mobj.group('options'),
2645                                                  video_id=video_id,
2646                                                  transform_source=transform_source)
2647             except ExtractorError:
2648                 pass
2649             else:
2650                 if isinstance(jwplayer_data, dict):
2651                     return jwplayer_data
2652
2653     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
2654         jwplayer_data = self._find_jwplayer_data(
2655             webpage, video_id, transform_source=js_to_json)
2656         return self._parse_jwplayer_data(
2657             jwplayer_data, video_id, *args, **kwargs)
2658
2659     def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
2660                              m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2661         # JWPlayer backward compatibility: flattened playlists
2662         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
2663         if 'playlist' not in jwplayer_data:
2664             jwplayer_data = {'playlist': [jwplayer_data]}
2665
2666         entries = []
2667
2668         # JWPlayer backward compatibility: single playlist item
2669         # https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
2670         if not isinstance(jwplayer_data['playlist'], list):
2671             jwplayer_data['playlist'] = [jwplayer_data['playlist']]
2672
2673         for video_data in jwplayer_data['playlist']:
2674             # JWPlayer backward compatibility: flattened sources
2675             # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
2676             if 'sources' not in video_data:
2677                 video_data['sources'] = [video_data]
2678
2679             this_video_id = video_id or video_data['mediaid']
2680
2681             formats = self._parse_jwplayer_formats(
2682                 video_data['sources'], video_id=this_video_id, m3u8_id=m3u8_id,
2683                 mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
2684
2685             subtitles = {}
2686             tracks = video_data.get('tracks')
2687             if tracks and isinstance(tracks, list):
2688                 for track in tracks:
2689                     if not isinstance(track, dict):
2690                         continue
2691                     track_kind = track.get('kind')
2692                     if not track_kind or not isinstance(track_kind, compat_str):
2693                         continue
2694                     if track_kind.lower() not in ('captions', 'subtitles'):
2695                         continue
2696                     track_url = urljoin(base_url, track.get('file'))
2697                     if not track_url:
2698                         continue
2699                     subtitles.setdefault(track.get('label') or 'en', []).append({
2700                         'url': self._proto_relative_url(track_url)
2701                     })
2702
2703             entry = {
2704                 'id': this_video_id,
2705                 'title': unescapeHTML(video_data['title'] if require_title else video_data.get('title')),
2706                 'description': clean_html(video_data.get('description')),
2707                 'thumbnail': urljoin(base_url, self._proto_relative_url(video_data.get('image'))),
2708                 'timestamp': int_or_none(video_data.get('pubdate')),
2709                 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
2710                 'subtitles': subtitles,
2711             }
2712             # https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
2713             if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
2714                 entry.update({
2715                     '_type': 'url_transparent',
2716                     'url': formats[0]['url'],
2717                 })
2718             else:
2719                 self._sort_formats(formats)
2720                 entry['formats'] = formats
2721             entries.append(entry)
2722         if len(entries) == 1:
2723             return entries[0]
2724         else:
2725             return self.playlist_result(entries)
2726
2727     def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
2728                                 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2729         urls = []
2730         formats = []
2731         for source in jwplayer_sources_data:
2732             if not isinstance(source, dict):
2733                 continue
2734             source_url = urljoin(
2735                 base_url, self._proto_relative_url(source.get('file')))
2736             if not source_url or source_url in urls:
2737                 continue
2738             urls.append(source_url)
2739             source_type = source.get('type') or ''
2740             ext = mimetype2ext(source_type) or determine_ext(source_url)
2741             if source_type == 'hls' or ext == 'm3u8':
2742                 formats.extend(self._extract_m3u8_formats(
2743                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
2744                     m3u8_id=m3u8_id, fatal=False))
2745             elif source_type == 'dash' or ext == 'mpd':
2746                 formats.extend(self._extract_mpd_formats(
2747                     source_url, video_id, mpd_id=mpd_id, fatal=False))
2748             elif ext == 'smil':
2749                 formats.extend(self._extract_smil_formats(
2750                     source_url, video_id, fatal=False))
2751             # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
2752             elif source_type.startswith('audio') or ext in (
2753                     'oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
2754                 formats.append({
2755                     'url': source_url,
2756                     'vcodec': 'none',
2757                     'ext': ext,
2758                 })
2759             else:
2760                 height = int_or_none(source.get('height'))
2761                 if height is None:
2762                     # Often no height is provided but there is a label in
2763                     # format like "1080p", "720p SD", or 1080.
2764                     height = int_or_none(self._search_regex(
2765                         r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
2766                         'height', default=None))
2767                 a_format = {
2768                     'url': source_url,
2769                     'width': int_or_none(source.get('width')),
2770                     'height': height,
2771                     'tbr': int_or_none(source.get('bitrate')),
2772                     'ext': ext,
2773                 }
2774                 if source_url.startswith('rtmp'):
2775                     a_format['ext'] = 'flv'
2776                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
2777                     # of jwplayer.flash.swf
2778                     rtmp_url_parts = re.split(
2779                         r'((?:mp4|mp3|flv):)', source_url, 1)
2780                     if len(rtmp_url_parts) == 3:
2781                         rtmp_url, prefix, play_path = rtmp_url_parts
2782                         a_format.update({
2783                             'url': rtmp_url,
2784                             'play_path': prefix + play_path,
2785                         })
2786                     if rtmp_params:
2787                         a_format.update(rtmp_params)
2788                 formats.append(a_format)
2789         return formats
2790
2791     def _live_title(self, name):
2792         """ Generate the title for a live video """
2793         now = datetime.datetime.now()
2794         now_str = now.strftime('%Y-%m-%d %H:%M')
2795         return name + ' ' + now_str
2796
2797     def _int(self, v, name, fatal=False, **kwargs):
2798         res = int_or_none(v, **kwargs)
2799         if 'get_attr' in kwargs:
2800             print(getattr(v, kwargs['get_attr']))
2801         if res is None:
2802             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2803             if fatal:
2804                 raise ExtractorError(msg)
2805             else:
2806                 self._downloader.report_warning(msg)
2807         return res
2808
2809     def _float(self, v, name, fatal=False, **kwargs):
2810         res = float_or_none(v, **kwargs)
2811         if res is None:
2812             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2813             if fatal:
2814                 raise ExtractorError(msg)
2815             else:
2816                 self._downloader.report_warning(msg)
2817         return res
2818
2819     def _set_cookie(self, domain, name, value, expire_time=None, port=None,
2820                     path='/', secure=False, discard=False, rest={}, **kwargs):
2821         cookie = compat_cookiejar.Cookie(
2822             0, name, value, port, port is not None, domain, True,
2823             domain.startswith('.'), path, True, secure, expire_time,
2824             discard, None, None, rest)
2825         self._downloader.cookiejar.set_cookie(cookie)
2826
2827     def _get_cookies(self, url):
2828         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
2829         req = sanitized_Request(url)
2830         self._downloader.cookiejar.add_cookie_header(req)
2831         return compat_cookies.SimpleCookie(req.get_header('Cookie'))
2832
2833     def _apply_first_set_cookie_header(self, url_handle, cookie):
2834         """
2835         Apply first Set-Cookie header instead of the last. Experimental.
2836
2837         Some sites (e.g. [1-3]) may serve two cookies under the same name
2838         in Set-Cookie header and expect the first (old) one to be set rather
2839         than second (new). However, as of RFC6265 the newer one cookie
2840         should be set into cookie store what actually happens.
2841         We will workaround this issue by resetting the cookie to
2842         the first one manually.
2843         1. https://new.vk.com/
2844         2. https://github.com/ytdl-org/youtube-dl/issues/9841#issuecomment-227871201
2845         3. https://learning.oreilly.com/
2846         """
2847         for header, cookies in url_handle.headers.items():
2848             if header.lower() != 'set-cookie':
2849                 continue
2850             if sys.version_info[0] >= 3:
2851                 cookies = cookies.encode('iso-8859-1')
2852             cookies = cookies.decode('utf-8')
2853             cookie_value = re.search(
2854                 r'%s=(.+?);.*?\b[Dd]omain=(.+?)(?:[,;]|$)' % cookie, cookies)
2855             if cookie_value:
2856                 value, domain = cookie_value.groups()
2857                 self._set_cookie(domain, cookie, value)
2858                 break
2859
2860     def get_testcases(self, include_onlymatching=False):
2861         t = getattr(self, '_TEST', None)
2862         if t:
2863             assert not hasattr(self, '_TESTS'), \
2864                 '%s has _TEST and _TESTS' % type(self).__name__
2865             tests = [t]
2866         else:
2867             tests = getattr(self, '_TESTS', [])
2868         for t in tests:
2869             if not include_onlymatching and t.get('only_matching', False):
2870                 continue
2871             t['name'] = type(self).__name__[:-len('IE')]
2872             yield t
2873
2874     def is_suitable(self, age_limit):
2875         """ Test whether the extractor is generally suitable for the given
2876         age limit (i.e. pornographic sites are not, all others usually are) """
2877
2878         any_restricted = False
2879         for tc in self.get_testcases(include_onlymatching=False):
2880             if tc.get('playlist', []):
2881                 tc = tc['playlist'][0]
2882             is_restricted = age_restricted(
2883                 tc.get('info_dict', {}).get('age_limit'), age_limit)
2884             if not is_restricted:
2885                 return True
2886             any_restricted = any_restricted or is_restricted
2887         return not any_restricted
2888
2889     def extract_subtitles(self, *args, **kwargs):
2890         if (self._downloader.params.get('writesubtitles', False)
2891                 or self._downloader.params.get('listsubtitles')):
2892             return self._get_subtitles(*args, **kwargs)
2893         return {}
2894
2895     def _get_subtitles(self, *args, **kwargs):
2896         raise NotImplementedError('This method must be implemented by subclasses')
2897
2898     @staticmethod
2899     def _merge_subtitle_items(subtitle_list1, subtitle_list2):
2900         """ Merge subtitle items for one language. Items with duplicated URLs
2901         will be dropped. """
2902         list1_urls = set([item['url'] for item in subtitle_list1])
2903         ret = list(subtitle_list1)
2904         ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
2905         return ret
2906
2907     @classmethod
2908     def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
2909         """ Merge two subtitle dictionaries, language by language. """
2910         ret = dict(subtitle_dict1)
2911         for lang in subtitle_dict2:
2912             ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
2913         return ret
2914
2915     def extract_automatic_captions(self, *args, **kwargs):
2916         if (self._downloader.params.get('writeautomaticsub', False)
2917                 or self._downloader.params.get('listsubtitles')):
2918             return self._get_automatic_captions(*args, **kwargs)
2919         return {}
2920
2921     def _get_automatic_captions(self, *args, **kwargs):
2922         raise NotImplementedError('This method must be implemented by subclasses')
2923
2924     def mark_watched(self, *args, **kwargs):
2925         if (self._downloader.params.get('mark_watched', False)
2926                 and (self._get_login_info()[0] is not None
2927                      or self._downloader.params.get('cookiefile') is not None)):
2928             self._mark_watched(*args, **kwargs)
2929
2930     def _mark_watched(self, *args, **kwargs):
2931         raise NotImplementedError('This method must be implemented by subclasses')
2932
2933     def geo_verification_headers(self):
2934         headers = {}
2935         geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
2936         if geo_verification_proxy:
2937             headers['Ytdl-request-proxy'] = geo_verification_proxy
2938         return headers
2939
2940     def _generic_id(self, url):
2941         return compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
2942
2943     def _generic_title(self, url):
2944         return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
2945
2946
2947 class SearchInfoExtractor(InfoExtractor):
2948     """
2949     Base class for paged search queries extractors.
2950     They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
2951     Instances should define _SEARCH_KEY and _MAX_RESULTS.
2952     """
2953
2954     @classmethod
2955     def _make_valid_url(cls):
2956         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
2957
2958     @classmethod
2959     def suitable(cls, url):
2960         return re.match(cls._make_valid_url(), url) is not None
2961
2962     def _real_extract(self, query):
2963         mobj = re.match(self._make_valid_url(), query)
2964         if mobj is None:
2965             raise ExtractorError('Invalid search query "%s"' % query)
2966
2967         prefix = mobj.group('prefix')
2968         query = mobj.group('query')
2969         if prefix == '':
2970             return self._get_n_results(query, 1)
2971         elif prefix == 'all':
2972             return self._get_n_results(query, self._MAX_RESULTS)
2973         else:
2974             n = int(prefix)
2975             if n <= 0:
2976                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
2977             elif n > self._MAX_RESULTS:
2978                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
2979                 n = self._MAX_RESULTS
2980             return self._get_n_results(query, n)
2981
2982     def _get_n_results(self, query, n):
2983         """Get a specified number of results for a query"""
2984         raise NotImplementedError('This method must be implemented by subclasses')
2985
2986     @property
2987     def SEARCH_KEY(self):
2988         return self._SEARCH_KEY