2 from __future__ import unicode_literals
17 from ..compat import (
20 compat_etree_fromstring,
26 compat_urllib_parse_unquote,
27 compat_urllib_parse_urlencode,
28 compat_urllib_request,
30 compat_xml_parse_error,
32 from ..downloader.f4m import (
34 remove_encrypted_media,
59 parse_m3u8_attributes,
76 class InfoExtractor(object):
77 """Information Extractor class.
79 Information extractors are the classes that, given a URL, extract
80 information about the video (or videos) the URL refers to. This
81 information includes the real video URL, the video title, author and
82 others. The information is stored in a dictionary which is then
83 passed to the YoutubeDL. The YoutubeDL processes this
84 information possibly downloading the video to the file system, among
85 other possible outcomes.
87 The type field determines the type of the result.
88 By far the most common value (and the default if _type is missing) is
89 "video", which indicates a single video.
91 For a video, the dictionaries must include the following fields:
94 title: Video title, unescaped.
96 Additionally, it must contain either a formats entry or a url one:
98 formats: A list of dictionaries for each format available, ordered
99 from worst to best quality.
102 * url Mandatory. The URL of the video file
104 The URL of the manifest file in case of
105 fragmented media (DASH, hls, hds)
106 * ext Will be calculated from URL if missing
107 * format A human-readable description of the format
108 ("mp4 container with h264/opus").
109 Calculated from the format_id, width, height.
110 and format_note fields if missing.
111 * format_id A short description of the format
112 ("mp4_h264_opus" or "19").
113 Technically optional, but strongly recommended.
114 * format_note Additional info about the format
115 ("3D" or "DASH video")
116 * width Width of the video, if known
117 * height Height of the video, if known
118 * resolution Textual description of width and height
119 * tbr Average bitrate of audio and video in KBit/s
120 * abr Average audio bitrate in KBit/s
121 * acodec Name of the audio codec in use
122 * asr Audio sampling rate in Hertz
123 * vbr Average video bitrate in KBit/s
125 * vcodec Name of the video codec in use
126 * container Name of the container format
127 * filesize The number of bytes, if known in advance
128 * filesize_approx An estimate for the number of bytes
129 * player_url SWF Player URL (used for rtmpdump).
130 * protocol The protocol that will be used for the actual
131 download, lower-case.
132 "http", "https", "rtsp", "rtmp", "rtmpe",
133 "m3u8", "m3u8_native" or "http_dash_segments".
135 Base URL for fragments. Each fragment's path
136 value (if present) will be relative to
138 * fragments A list of fragments of a fragmented media.
139 Each fragment entry must contain either an url
140 or a path. If an url is present it should be
141 considered by a client. Otherwise both path and
142 fragment_base_url must be present. Here is
143 the list of all potential fields:
144 * "url" - fragment's URL
145 * "path" - fragment's path relative to
147 * "duration" (optional, int or float)
148 * "filesize" (optional, int)
149 * preference Order number of this format. If this field is
150 present and not None, the formats get sorted
151 by this field, regardless of all other values.
152 -1 for default (order by other properties),
153 -2 or smaller for less than default.
154 < -1000 to hide the format (if there is
155 another one which is strictly better)
156 * language Language code, e.g. "de" or "en-US".
157 * language_preference Is this in the language mentioned in
159 10 if it's what the URL is about,
160 -1 for default (don't know),
161 -10 otherwise, other values reserved for now.
162 * quality Order number of the video quality of this
163 format, irrespective of the file format.
164 -1 for default (order by other properties),
165 -2 or smaller for less than default.
166 * source_preference Order number for this video source
167 (quality takes higher priority)
168 -1 for default (order by other properties),
169 -2 or smaller for less than default.
170 * http_headers A dictionary of additional HTTP headers
171 to add to the request.
172 * stretched_ratio If given and not 1, indicates that the
173 video's pixels are not square.
174 width : height ratio as float.
175 * no_resume The server does not support resuming the
176 (HTTP or RTMP) download. Boolean.
177 * downloader_options A dictionary of downloader options as
178 described in FileDownloader
180 url: Final video URL.
181 ext: Video filename extension.
182 format: The video format, defaults to ext (used for --get-format)
183 player_url: SWF Player URL (used for rtmpdump).
185 The following fields are optional:
187 alt_title: A secondary title of the video.
188 display_id An alternative identifier for the video, not necessarily
189 unique, but available before title. Typically, id is
190 something like "4234987", title "Dancing naked mole rats",
191 and display_id "dancing-naked-mole-rats"
192 thumbnails: A list of dictionaries, with the following entries:
193 * "id" (optional, string) - Thumbnail format ID
195 * "preference" (optional, int) - quality of the image
196 * "width" (optional, int)
197 * "height" (optional, int)
198 * "resolution" (optional, string "{width}x{height"},
200 * "filesize" (optional, int)
201 thumbnail: Full URL to a video thumbnail image.
202 description: Full video description.
203 uploader: Full name of the video uploader.
204 license: License name the video is licensed under.
205 creator: The creator of the video.
206 release_date: The date (YYYYMMDD) when the video was released.
207 timestamp: UNIX timestamp of the moment the video became available.
208 upload_date: Video upload date (YYYYMMDD).
209 If not explicitly set, calculated from timestamp.
210 uploader_id: Nickname or id of the video uploader.
211 uploader_url: Full URL to a personal webpage of the video uploader.
212 location: Physical location where the video was filmed.
213 subtitles: The available subtitles as a dictionary in the format
214 {tag: subformats}. "tag" is usually a language code, and
215 "subformats" is a list sorted from lower to higher
216 preference, each element is a dictionary with the "ext"
218 * "data": The subtitles file contents
219 * "url": A URL pointing to the subtitles file
220 "ext" will be calculated from URL if missing
221 automatic_captions: Like 'subtitles', used by the YoutubeIE for
222 automatically generated captions
223 duration: Length of the video in seconds, as an integer or float.
224 view_count: How many users have watched the video on the platform.
225 like_count: Number of positive ratings of the video
226 dislike_count: Number of negative ratings of the video
227 repost_count: Number of reposts of the video
228 average_rating: Average rating give by users, the scale used depends on the webpage
229 comment_count: Number of comments on the video
230 comments: A list of comments, each with one or more of the following
231 properties (all but one of text or html optional):
232 * "author" - human-readable name of the comment author
233 * "author_id" - user ID of the comment author
235 * "html" - Comment as HTML
236 * "text" - Plain text of the comment
237 * "timestamp" - UNIX timestamp of comment
238 * "parent" - ID of the comment this one is replying to.
239 Set to "root" to indicate that this is a
240 comment to the original video.
241 age_limit: Age restriction for the video, as an integer (years)
242 webpage_url: The URL to the video webpage, if given to youtube-dl it
243 should allow to get the same result again. (It will be set
244 by YoutubeDL if it's missing)
245 categories: A list of categories that the video falls in, for example
247 tags: A list of tags assigned to the video, e.g. ["sweden", "pop music"]
248 is_live: True, False, or None (=unknown). Whether this video is a
249 live stream that goes on instead of a fixed-length video.
250 start_time: Time in seconds where the reproduction should start, as
251 specified in the URL.
252 end_time: Time in seconds where the reproduction should end, as
253 specified in the URL.
254 chapters: A list of dictionaries, with the following entries:
255 * "start_time" - The start time of the chapter in seconds
256 * "end_time" - The end time of the chapter in seconds
257 * "title" (optional, string)
259 The following fields should only be used when the video belongs to some logical
262 chapter: Name or title of the chapter the video belongs to.
263 chapter_number: Number of the chapter the video belongs to, as an integer.
264 chapter_id: Id of the chapter the video belongs to, as a unicode string.
266 The following fields should only be used when the video is an episode of some
267 series, programme or podcast:
269 series: Title of the series or programme the video episode belongs to.
270 season: Title of the season the video episode belongs to.
271 season_number: Number of the season the video episode belongs to, as an integer.
272 season_id: Id of the season the video episode belongs to, as a unicode string.
273 episode: Title of the video episode. Unlike mandatory video title field,
274 this field should denote the exact title of the video episode
275 without any kind of decoration.
276 episode_number: Number of the video episode within a season, as an integer.
277 episode_id: Id of the video episode, as a unicode string.
279 The following fields should only be used when the media is a track or a part of
282 track: Title of the track.
283 track_number: Number of the track within an album or a disc, as an integer.
284 track_id: Id of the track (useful in case of custom indexing, e.g. 6.iii),
286 artist: Artist(s) of the track.
287 genre: Genre(s) of the track.
288 album: Title of the album the track belongs to.
289 album_type: Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc).
290 album_artist: List of all artists appeared on the album (e.g.
291 "Ash Borer / Fell Voices" or "Various Artists", useful for splits
293 disc_number: Number of the disc or other physical medium the track belongs to,
295 release_year: Year (YYYY) when the album was released.
297 Unless mentioned otherwise, the fields should be Unicode strings.
299 Unless mentioned otherwise, None is equivalent to absence of information.
302 _type "playlist" indicates multiple videos.
303 There must be a key "entries", which is a list, an iterable, or a PagedList
304 object, each element of which is a valid dictionary by this specification.
306 Additionally, playlists can have "id", "title", "description", "uploader",
307 "uploader_id", "uploader_url" attributes with the same semantics as videos
311 _type "multi_video" indicates that there are multiple videos that
312 form a single show, for examples multiple acts of an opera or TV episode.
313 It must have an entries key like a playlist and contain all the keys
314 required for a video at the same time.
317 _type "url" indicates that the video must be extracted from another
318 location, possibly by a different extractor. Its only required key is:
319 "url" - the next URL to extract.
320 The key "ie_key" can be set to the class name (minus the trailing "IE",
321 e.g. "Youtube") if the extractor class is known in advance.
322 Additionally, the dictionary may have any properties of the resolved entity
323 known in advance, for example "title" if the title of the referred video is
327 _type "url_transparent" entities have the same specification as "url", but
328 indicate that the given additional information is more precise than the one
329 associated with the resolved URL.
330 This is useful when a site employs a video service that hosts the video and
331 its technical metadata, but that video service does not embed a useful
332 title, description etc.
335 Subclasses of this one should re-define the _real_initialize() and
336 _real_extract() methods and define a _VALID_URL regexp.
337 Probably, they should also be added to the list of extractors.
339 _GEO_BYPASS attribute may be set to False in order to disable
340 geo restriction bypass mechanisms for a particular extractor.
341 Though it won't disable explicit geo restriction bypass based on
342 country code provided with geo_bypass_country. (experimental)
344 _GEO_COUNTRIES attribute may contain a list of presumably geo unrestricted
345 countries for this extractor. One of these countries will be used by
346 geo restriction bypass mechanism right away in order to bypass
347 geo restriction, of course, if the mechanism is not disabled. (experimental)
349 NB: both these geo attributes are experimental and may change in future
350 or be completely removed.
352 Finally, the _WORKING attribute should be set to False for broken IEs
353 in order to warn the users and skip the tests.
358 _x_forwarded_for_ip = None
360 _GEO_COUNTRIES = None
363 def __init__(self, downloader=None):
364 """Constructor. Receives an optional downloader."""
366 self._x_forwarded_for_ip = None
367 self.set_downloader(downloader)
370 def suitable(cls, url):
371 """Receives a URL and returns True if suitable for this IE."""
373 # This does not use has/getattr intentionally - we want to know whether
374 # we have cached the regexp for *this* class, whereas getattr would also
375 # match the superclass
376 if '_VALID_URL_RE' not in cls.__dict__:
377 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
378 return cls._VALID_URL_RE.match(url) is not None
381 def _match_id(cls, url):
382 if '_VALID_URL_RE' not in cls.__dict__:
383 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
384 m = cls._VALID_URL_RE.match(url)
386 return compat_str(m.group('id'))
390 """Getter method for _WORKING."""
393 def initialize(self):
394 """Initializes an instance (authentication, etc)."""
395 self._initialize_geo_bypass(self._GEO_COUNTRIES)
397 self._real_initialize()
400 def _initialize_geo_bypass(self, countries):
402 Initialize geo restriction bypass mechanism.
404 This method is used to initialize geo bypass mechanism based on faking
405 X-Forwarded-For HTTP header. A random country from provided country list
406 is selected and a random IP belonging to this country is generated. This
407 IP will be passed as X-Forwarded-For HTTP header in all subsequent
410 This method will be used for initial geo bypass mechanism initialization
411 during the instance initialization with _GEO_COUNTRIES.
413 You may also manually call it from extractor's code if geo countries
414 information is not available beforehand (e.g. obtained during
415 extraction) or due to some another reason.
417 if not self._x_forwarded_for_ip:
418 country_code = self._downloader.params.get('geo_bypass_country', None)
419 # If there is no explicit country for geo bypass specified and
420 # the extractor is known to be geo restricted let's fake IP
421 # as X-Forwarded-For right away.
422 if (not country_code and
424 self._downloader.params.get('geo_bypass', True) and
426 country_code = random.choice(countries)
428 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
429 if self._downloader.params.get('verbose', False):
430 self._downloader.to_screen(
431 '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
432 % (self._x_forwarded_for_ip, country_code.upper()))
434 def extract(self, url):
435 """Extracts URL information and returns it in list of dicts."""
440 ie_result = self._real_extract(url)
441 if self._x_forwarded_for_ip:
442 ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
444 except GeoRestrictedError as e:
445 if self.__maybe_fake_ip_and_retry(e.countries):
448 except ExtractorError:
450 except compat_http_client.IncompleteRead as e:
451 raise ExtractorError('A network error has occurred.', cause=e, expected=True)
452 except (KeyError, StopIteration) as e:
453 raise ExtractorError('An extractor error has occurred.', cause=e)
455 def __maybe_fake_ip_and_retry(self, countries):
456 if (not self._downloader.params.get('geo_bypass_country', None) and
458 self._downloader.params.get('geo_bypass', True) and
459 not self._x_forwarded_for_ip and
461 country_code = random.choice(countries)
462 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
463 if self._x_forwarded_for_ip:
465 'Video is geo restricted. Retrying extraction with fake IP %s (%s) as X-Forwarded-For.'
466 % (self._x_forwarded_for_ip, country_code.upper()))
470 def set_downloader(self, downloader):
471 """Sets the downloader for this IE."""
472 self._downloader = downloader
474 def _real_initialize(self):
475 """Real initialization process. Redefine in subclasses."""
478 def _real_extract(self, url):
479 """Real extraction process. Redefine in subclasses."""
484 """A string for getting the InfoExtractor with get_info_extractor"""
485 return compat_str(cls.__name__[:-2])
489 return compat_str(type(self).__name__[:-2])
491 def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
492 """ Returns the response handle """
494 self.report_download_webpage(video_id)
495 elif note is not False:
497 self.to_screen('%s' % (note,))
499 self.to_screen('%s: %s' % (video_id, note))
501 # Some sites check X-Forwarded-For HTTP header in order to figure out
502 # the origin of the client behind proxy. This allows bypassing geo
503 # restriction by faking this header's value to IP that belongs to some
504 # geo unrestricted country. We will do so once we encounter any
505 # geo restriction error.
506 if self._x_forwarded_for_ip:
507 if 'X-Forwarded-For' not in headers:
508 headers['X-Forwarded-For'] = self._x_forwarded_for_ip
510 if isinstance(url_or_request, compat_urllib_request.Request):
511 url_or_request = update_Request(
512 url_or_request, data=data, headers=headers, query=query)
515 url_or_request = update_url_query(url_or_request, query)
516 if data is not None or headers:
517 url_or_request = sanitized_Request(url_or_request, data, headers)
519 return self._downloader.urlopen(url_or_request)
520 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
524 errnote = 'Unable to download webpage'
526 errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
528 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
530 self._downloader.report_warning(errmsg)
533 def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}):
534 """ Returns a tuple (page content as string, URL handle) """
535 # Strip hashes from the URL (#1038)
536 if isinstance(url_or_request, (compat_str, str)):
537 url_or_request = url_or_request.partition('#')[0]
539 urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
543 content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
544 return (content, urlh)
547 def _guess_encoding_from_content(content_type, webpage_bytes):
548 m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
550 encoding = m.group(1)
552 m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
553 webpage_bytes[:1024])
555 encoding = m.group(1).decode('ascii')
556 elif webpage_bytes.startswith(b'\xff\xfe'):
563 def __check_blocked(self, content):
564 first_block = content[:512]
565 if ('<title>Access to this site is blocked</title>' in content and
566 'Websense' in first_block):
567 msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
568 blocked_iframe = self._html_search_regex(
569 r'<iframe src="([^"]+)"', content,
570 'Websense information URL', default=None)
572 msg += ' Visit %s for more details' % blocked_iframe
573 raise ExtractorError(msg, expected=True)
574 if '<title>The URL you requested has been blocked</title>' in first_block:
576 'Access to this webpage has been blocked by Indian censorship. '
577 'Use a VPN or proxy server (with --proxy) to route around it.')
578 block_msg = self._html_search_regex(
579 r'</h1><p>(.*?)</p>',
580 content, 'block message', default=None)
582 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
583 raise ExtractorError(msg, expected=True)
584 if ('<title>TTK :: Доступ к ресурсу ограничен</title>' in content and
585 'blocklist.rkn.gov.ru' in content):
586 raise ExtractorError(
587 'Access to this webpage has been blocked by decision of the Russian government. '
588 'Visit http://blocklist.rkn.gov.ru/ for a block reason.',
591 def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
592 content_type = urlh.headers.get('Content-Type', '')
593 webpage_bytes = urlh.read()
594 if prefix is not None:
595 webpage_bytes = prefix + webpage_bytes
597 encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
598 if self._downloader.params.get('dump_intermediate_pages', False):
599 self.to_screen('Dumping request to ' + urlh.geturl())
600 dump = base64.b64encode(webpage_bytes).decode('ascii')
601 self._downloader.to_screen(dump)
602 if self._downloader.params.get('write_pages', False):
603 basen = '%s_%s' % (video_id, urlh.geturl())
605 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
606 basen = basen[:240 - len(h)] + h
607 raw_filename = basen + '.dump'
608 filename = sanitize_filename(raw_filename, restricted=True)
609 self.to_screen('Saving request to ' + filename)
610 # Working around MAX_PATH limitation on Windows (see
611 # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
612 if compat_os_name == 'nt':
613 absfilepath = os.path.abspath(filename)
614 if len(absfilepath) > 259:
615 filename = '\\\\?\\' + absfilepath
616 with open(filename, 'wb') as outf:
617 outf.write(webpage_bytes)
620 content = webpage_bytes.decode(encoding, 'replace')
622 content = webpage_bytes.decode('utf-8', 'replace')
624 self.__check_blocked(content)
628 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None, data=None, headers={}, query={}):
629 """ Returns the data of the page as a string """
632 while success is False:
634 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding, data=data, headers=headers, query=query)
636 except compat_http_client.IncompleteRead as e:
638 if try_count >= tries:
640 self._sleep(timeout, video_id)
647 def _download_xml_handle(
648 self, url_or_request, video_id, note='Downloading XML',
649 errnote='Unable to download XML', transform_source=None,
650 fatal=True, encoding=None, data=None, headers={}, query={}):
651 """Return a tuple (xml as an xml.etree.ElementTree.Element, URL handle)"""
652 res = self._download_webpage_handle(
653 url_or_request, video_id, note, errnote, fatal=fatal,
654 encoding=encoding, data=data, headers=headers, query=query)
657 xml_string, urlh = res
658 return self._parse_xml(
659 xml_string, video_id, transform_source=transform_source,
662 def _download_xml(self, url_or_request, video_id,
663 note='Downloading XML', errnote='Unable to download XML',
664 transform_source=None, fatal=True, encoding=None,
665 data=None, headers={}, query={}):
666 """Return the xml as an xml.etree.ElementTree.Element"""
667 res = self._download_xml_handle(
668 url_or_request, video_id, note=note, errnote=errnote,
669 transform_source=transform_source, fatal=fatal, encoding=encoding,
670 data=data, headers=headers, query=query)
671 return res if res is False else res[0]
673 def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
675 xml_string = transform_source(xml_string)
677 return compat_etree_fromstring(xml_string.encode('utf-8'))
678 except compat_xml_parse_error as ve:
679 errmsg = '%s: Failed to parse XML ' % video_id
681 raise ExtractorError(errmsg, cause=ve)
683 self.report_warning(errmsg + str(ve))
685 def _download_json_handle(
686 self, url_or_request, video_id, note='Downloading JSON metadata',
687 errnote='Unable to download JSON metadata', transform_source=None,
688 fatal=True, encoding=None, data=None, headers={}, query={}):
689 """Return a tuple (JSON object, URL handle)"""
690 res = self._download_webpage_handle(
691 url_or_request, video_id, note, errnote, fatal=fatal,
692 encoding=encoding, data=data, headers=headers, query=query)
695 json_string, urlh = res
696 return self._parse_json(
697 json_string, video_id, transform_source=transform_source,
701 self, url_or_request, video_id, note='Downloading JSON metadata',
702 errnote='Unable to download JSON metadata', transform_source=None,
703 fatal=True, encoding=None, data=None, headers={}, query={}):
704 res = self._download_json_handle(
705 url_or_request, video_id, note=note, errnote=errnote,
706 transform_source=transform_source, fatal=fatal, encoding=encoding,
707 data=data, headers=headers, query=query)
708 return res if res is False else res[0]
710 def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
712 json_string = transform_source(json_string)
714 return json.loads(json_string)
715 except ValueError as ve:
716 errmsg = '%s: Failed to parse JSON ' % video_id
718 raise ExtractorError(errmsg, cause=ve)
720 self.report_warning(errmsg + str(ve))
722 def report_warning(self, msg, video_id=None):
723 idstr = '' if video_id is None else '%s: ' % video_id
724 self._downloader.report_warning(
725 '[%s] %s%s' % (self.IE_NAME, idstr, msg))
727 def to_screen(self, msg):
728 """Print msg to screen, prefixing it with '[ie_name]'"""
729 self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
731 def report_extraction(self, id_or_name):
732 """Report information extraction."""
733 self.to_screen('%s: Extracting information' % id_or_name)
735 def report_download_webpage(self, video_id):
736 """Report webpage download."""
737 self.to_screen('%s: Downloading webpage' % video_id)
739 def report_age_confirmation(self):
740 """Report attempt to confirm age."""
741 self.to_screen('Confirming age')
743 def report_login(self):
744 """Report attempt to log in."""
745 self.to_screen('Logging in')
748 def raise_login_required(msg='This video is only available for registered users'):
749 raise ExtractorError(
750 '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
754 def raise_geo_restricted(msg='This video is not available from your location due to geo restriction', countries=None):
755 raise GeoRestrictedError(msg, countries=countries)
757 # Methods for following #608
759 def url_result(url, ie=None, video_id=None, video_title=None):
760 """Returns a URL that points to a page that should be processed"""
761 # TODO: ie should be the class used for getting the info
762 video_info = {'_type': 'url',
765 if video_id is not None:
766 video_info['id'] = video_id
767 if video_title is not None:
768 video_info['title'] = video_title
771 def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None):
773 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
775 return self.playlist_result(
776 urls, playlist_id=playlist_id, playlist_title=playlist_title)
779 def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
780 """Returns a playlist"""
781 video_info = {'_type': 'playlist',
784 video_info['id'] = playlist_id
786 video_info['title'] = playlist_title
787 if playlist_description:
788 video_info['description'] = playlist_description
791 def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
793 Perform a regex search on the given string, using a single or a list of
794 patterns returning the first matching group.
795 In case of failure return a default value or raise a WARNING or a
796 RegexNotFoundError, depending on fatal, specifying the field name.
798 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
799 mobj = re.search(pattern, string, flags)
802 mobj = re.search(p, string, flags)
806 if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
807 _name = '\033[0;34m%s\033[0m' % name
813 # return the first matching group
814 return next(g for g in mobj.groups() if g is not None)
816 return mobj.group(group)
817 elif default is not NO_DEFAULT:
820 raise RegexNotFoundError('Unable to extract %s' % _name)
822 self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
825 def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
827 Like _search_regex, but strips HTML tags and unescapes entities.
829 res = self._search_regex(pattern, string, name, default, fatal, flags, group)
831 return clean_html(res).strip()
835 def _get_netrc_login_info(self, netrc_machine=None):
838 netrc_machine = netrc_machine or self._NETRC_MACHINE
840 if self._downloader.params.get('usenetrc', False):
842 info = netrc.netrc().authenticators(netrc_machine)
847 raise netrc.NetrcParseError(
848 'No authenticators for %s' % netrc_machine)
849 except (IOError, netrc.NetrcParseError) as err:
850 self._downloader.report_warning(
851 'parsing .netrc: %s' % error_to_compat_str(err))
853 return username, password
855 def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
857 Get the login info as (username, password)
858 First look for the manually specified credentials using username_option
859 and password_option as keys in params dictionary. If no such credentials
860 available look in the netrc file using the netrc_machine or _NETRC_MACHINE
862 If there's no info available, return (None, None)
864 if self._downloader is None:
867 downloader_params = self._downloader.params
869 # Attempt to use provided username and password or .netrc data
870 if downloader_params.get(username_option) is not None:
871 username = downloader_params[username_option]
872 password = downloader_params[password_option]
874 username, password = self._get_netrc_login_info(netrc_machine)
876 return username, password
878 def _get_tfa_info(self, note='two-factor verification code'):
880 Get the two-factor authentication info
881 TODO - asking the user will be required for sms/phone verify
882 currently just uses the command line option
883 If there's no info available, return None
885 if self._downloader is None:
887 downloader_params = self._downloader.params
889 if downloader_params.get('twofactor') is not None:
890 return downloader_params['twofactor']
892 return compat_getpass('Type %s and press [Return]: ' % note)
894 # Helper functions for extracting OpenGraph info
896 def _og_regexes(prop):
897 content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
898 property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
899 % {'prop': re.escape(prop)})
900 template = r'<meta[^>]+?%s[^>]+?%s'
902 template % (property_re, content_re),
903 template % (content_re, property_re),
907 def _meta_regex(prop):
908 return r'''(?isx)<meta
909 (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
910 [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
912 def _og_search_property(self, prop, html, name=None, **kargs):
913 if not isinstance(prop, (list, tuple)):
916 name = 'OpenGraph %s' % prop[0]
919 og_regexes.extend(self._og_regexes(p))
920 escaped = self._search_regex(og_regexes, html, name, flags=re.DOTALL, **kargs)
923 return unescapeHTML(escaped)
925 def _og_search_thumbnail(self, html, **kargs):
926 return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
928 def _og_search_description(self, html, **kargs):
929 return self._og_search_property('description', html, fatal=False, **kargs)
931 def _og_search_title(self, html, **kargs):
932 return self._og_search_property('title', html, **kargs)
934 def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
935 regexes = self._og_regexes('video') + self._og_regexes('video:url')
937 regexes = self._og_regexes('video:secure_url') + regexes
938 return self._html_search_regex(regexes, html, name, **kargs)
940 def _og_search_url(self, html, **kargs):
941 return self._og_search_property('url', html, **kargs)
943 def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
944 if not isinstance(name, (list, tuple)):
946 if display_name is None:
947 display_name = name[0]
948 return self._html_search_regex(
949 [self._meta_regex(n) for n in name],
950 html, display_name, fatal=fatal, group='content', **kwargs)
952 def _dc_search_uploader(self, html):
953 return self._html_search_meta('dc.creator', html, 'uploader')
955 def _rta_search(self, html):
956 # See http://www.rtalabel.org/index.php?content=howtofaq#single
957 if re.search(r'(?ix)<meta\s+name="rating"\s+'
958 r' content="RTA-5042-1996-1400-1577-RTA"',
963 def _media_rating_search(self, html):
964 # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
965 rating = self._html_search_meta('rating', html)
977 return RATING_TABLE.get(rating.lower())
979 def _family_friendly_search(self, html):
980 # See http://schema.org/VideoObject
981 family_friendly = self._html_search_meta(
982 'isFamilyFriendly', html, default=None)
984 if not family_friendly:
993 return RATING_TABLE.get(family_friendly.lower())
995 def _twitter_search_player(self, html):
996 return self._html_search_meta('twitter:player', html,
997 'twitter card player')
999 def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
1000 json_ld = self._search_regex(
1001 r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
1002 html, 'JSON-LD', group='json_ld', **kwargs)
1003 default = kwargs.get('default', NO_DEFAULT)
1005 return default if default is not NO_DEFAULT else {}
1006 # JSON-LD may be malformed and thus `fatal` should be respected.
1007 # At the same time `default` may be passed that assumes `fatal=False`
1008 # for _search_regex. Let's simulate the same behavior here as well.
1009 fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
1010 return self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)
1012 def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
1013 if isinstance(json_ld, compat_str):
1014 json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
1018 if not isinstance(json_ld, (list, tuple, dict)):
1020 if isinstance(json_ld, dict):
1023 INTERACTION_TYPE_MAP = {
1024 'CommentAction': 'comment',
1025 'AgreeAction': 'like',
1026 'DisagreeAction': 'dislike',
1027 'LikeAction': 'like',
1028 'DislikeAction': 'dislike',
1029 'ListenAction': 'view',
1030 'WatchAction': 'view',
1031 'ViewAction': 'view',
1034 def extract_interaction_statistic(e):
1035 interaction_statistic = e.get('interactionStatistic')
1036 if not isinstance(interaction_statistic, list):
1038 for is_e in interaction_statistic:
1039 if not isinstance(is_e, dict):
1041 if is_e.get('@type') != 'InteractionCounter':
1043 interaction_type = is_e.get('interactionType')
1044 if not isinstance(interaction_type, compat_str):
1046 interaction_count = int_or_none(is_e.get('userInteractionCount'))
1047 if interaction_count is None:
1049 count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1])
1052 count_key = '%s_count' % count_kind
1053 if info.get(count_key) is not None:
1055 info[count_key] = interaction_count
1057 def extract_video_object(e):
1058 assert e['@type'] == 'VideoObject'
1060 'url': e.get('contentUrl'),
1061 'title': unescapeHTML(e.get('name')),
1062 'description': unescapeHTML(e.get('description')),
1063 'thumbnail': e.get('thumbnailUrl') or e.get('thumbnailURL'),
1064 'duration': parse_duration(e.get('duration')),
1065 'timestamp': unified_timestamp(e.get('uploadDate')),
1066 'filesize': float_or_none(e.get('contentSize')),
1067 'tbr': int_or_none(e.get('bitrate')),
1068 'width': int_or_none(e.get('width')),
1069 'height': int_or_none(e.get('height')),
1070 'view_count': int_or_none(e.get('interactionCount')),
1072 extract_interaction_statistic(e)
1075 if isinstance(e.get('@context'), compat_str) and re.match(r'^https?://schema.org/?$', e.get('@context')):
1076 item_type = e.get('@type')
1077 if expected_type is not None and expected_type != item_type:
1079 if item_type in ('TVEpisode', 'Episode'):
1081 'episode': unescapeHTML(e.get('name')),
1082 'episode_number': int_or_none(e.get('episodeNumber')),
1083 'description': unescapeHTML(e.get('description')),
1085 part_of_season = e.get('partOfSeason')
1086 if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
1087 info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
1088 part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
1089 if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
1090 info['series'] = unescapeHTML(part_of_series.get('name'))
1091 elif item_type in ('Article', 'NewsArticle'):
1093 'timestamp': parse_iso8601(e.get('datePublished')),
1094 'title': unescapeHTML(e.get('headline')),
1095 'description': unescapeHTML(e.get('articleBody')),
1097 elif item_type == 'VideoObject':
1098 extract_video_object(e)
1100 video = e.get('video')
1101 if isinstance(video, dict) and video.get('@type') == 'VideoObject':
1102 extract_video_object(video)
1104 return dict((k, v) for k, v in info.items() if v is not None)
1107 def _hidden_inputs(html):
1108 html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
1110 for input in re.findall(r'(?i)(<input[^>]+>)', html):
1111 attrs = extract_attributes(input)
1114 if attrs.get('type') not in ('hidden', 'submit'):
1116 name = attrs.get('name') or attrs.get('id')
1117 value = attrs.get('value')
1118 if name and value is not None:
1119 hidden_inputs[name] = value
1120 return hidden_inputs
1122 def _form_hidden_inputs(self, form_id, html):
1123 form = self._search_regex(
1124 r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
1125 html, '%s form' % form_id, group='form')
1126 return self._hidden_inputs(form)
1128 def _sort_formats(self, formats, field_preference=None):
1130 raise ExtractorError('No video formats found')
1133 # Automatically determine tbr when missing based on abr and vbr (improves
1134 # formats sorting in some cases)
1135 if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
1136 f['tbr'] = f['abr'] + f['vbr']
1138 def _formats_key(f):
1139 # TODO remove the following workaround
1140 from ..utils import determine_ext
1141 if not f.get('ext') and 'url' in f:
1142 f['ext'] = determine_ext(f['url'])
1144 if isinstance(field_preference, (list, tuple)):
1147 if f.get(field) is not None
1148 else ('' if field == 'format_id' else -1)
1149 for field in field_preference)
1151 preference = f.get('preference')
1152 if preference is None:
1154 if f.get('ext') in ['f4f', 'f4m']: # Not yet supported
1157 protocol = f.get('protocol') or determine_protocol(f)
1158 proto_preference = 0 if protocol in ['http', 'https'] else (-0.5 if protocol == 'rtsp' else -0.1)
1160 if f.get('vcodec') == 'none': # audio only
1162 if self._downloader.params.get('prefer_free_formats'):
1163 ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
1165 ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
1168 audio_ext_preference = ORDER.index(f['ext'])
1170 audio_ext_preference = -1
1172 if f.get('acodec') == 'none': # video only
1174 if self._downloader.params.get('prefer_free_formats'):
1175 ORDER = ['flv', 'mp4', 'webm']
1177 ORDER = ['webm', 'flv', 'mp4']
1179 ext_preference = ORDER.index(f['ext'])
1182 audio_ext_preference = 0
1186 f.get('language_preference') if f.get('language_preference') is not None else -1,
1187 f.get('quality') if f.get('quality') is not None else -1,
1188 f.get('tbr') if f.get('tbr') is not None else -1,
1189 f.get('filesize') if f.get('filesize') is not None else -1,
1190 f.get('vbr') if f.get('vbr') is not None else -1,
1191 f.get('height') if f.get('height') is not None else -1,
1192 f.get('width') if f.get('width') is not None else -1,
1195 f.get('abr') if f.get('abr') is not None else -1,
1196 audio_ext_preference,
1197 f.get('fps') if f.get('fps') is not None else -1,
1198 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
1199 f.get('source_preference') if f.get('source_preference') is not None else -1,
1200 f.get('format_id') if f.get('format_id') is not None else '',
1202 formats.sort(key=_formats_key)
1204 def _check_formats(self, formats, video_id):
1206 formats[:] = filter(
1207 lambda f: self._is_valid_url(
1209 item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
1213 def _remove_duplicate_formats(formats):
1217 if f['url'] not in format_urls:
1218 format_urls.add(f['url'])
1219 unique_formats.append(f)
1220 formats[:] = unique_formats
1222 def _is_valid_url(self, url, video_id, item='video', headers={}):
1223 url = self._proto_relative_url(url, scheme='http:')
1224 # For now assume non HTTP(S) URLs always valid
1225 if not (url.startswith('http://') or url.startswith('https://')):
1228 self._request_webpage(url, video_id, 'Checking %s URL' % item, headers=headers)
1230 except ExtractorError as e:
1231 if isinstance(e.cause, compat_urllib_error.URLError):
1233 '%s: %s URL is invalid, skipping' % (video_id, item))
1237 def http_scheme(self):
1238 """ Either "http:" or "https:", depending on the user's preferences """
1241 if self._downloader.params.get('prefer_insecure', False)
1244 def _proto_relative_url(self, url, scheme=None):
1247 if url.startswith('//'):
1249 scheme = self.http_scheme()
1254 def _sleep(self, timeout, video_id, msg_template=None):
1255 if msg_template is None:
1256 msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
1257 msg = msg_template % {'video_id': video_id, 'timeout': timeout}
1261 def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
1262 transform_source=lambda s: fix_xml_ampersands(s).strip(),
1263 fatal=True, m3u8_id=None):
1264 manifest = self._download_xml(
1265 manifest_url, video_id, 'Downloading f4m manifest',
1266 'Unable to download f4m manifest',
1267 # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
1268 # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
1269 transform_source=transform_source,
1272 if manifest is False:
1275 return self._parse_f4m_formats(
1276 manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1277 transform_source=transform_source, fatal=fatal, m3u8_id=m3u8_id)
1279 def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
1280 transform_source=lambda s: fix_xml_ampersands(s).strip(),
1281 fatal=True, m3u8_id=None):
1282 # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
1283 akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
1284 if akamai_pv is not None and ';' in akamai_pv.text:
1285 playerVerificationChallenge = akamai_pv.text.split(';')[0]
1286 if playerVerificationChallenge.strip() != '':
1290 manifest_version = '1.0'
1291 media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
1293 manifest_version = '2.0'
1294 media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
1295 # Remove unsupported DRM protected media from final formats
1296 # rendition (see https://github.com/rg3/youtube-dl/issues/8573).
1297 media_nodes = remove_encrypted_media(media_nodes)
1301 manifest_base_url = get_base_url(manifest)
1303 bootstrap_info = xpath_element(
1304 manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
1305 'bootstrap info', default=None)
1308 mime_type = xpath_text(
1309 manifest, ['{http://ns.adobe.com/f4m/1.0}mimeType', '{http://ns.adobe.com/f4m/2.0}mimeType'],
1310 'base URL', default=None)
1311 if mime_type and mime_type.startswith('audio/'):
1314 for i, media_el in enumerate(media_nodes):
1315 tbr = int_or_none(media_el.attrib.get('bitrate'))
1316 width = int_or_none(media_el.attrib.get('width'))
1317 height = int_or_none(media_el.attrib.get('height'))
1318 format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
1319 # If <bootstrapInfo> is present, the specified f4m is a
1320 # stream-level manifest, and only set-level manifests may refer to
1321 # external resources. See section 11.4 and section 4 of F4M spec
1322 if bootstrap_info is None:
1324 # @href is introduced in 2.0, see section 11.6 of F4M spec
1325 if manifest_version == '2.0':
1326 media_url = media_el.attrib.get('href')
1327 if media_url is None:
1328 media_url = media_el.attrib.get('url')
1332 media_url if media_url.startswith('http://') or media_url.startswith('https://')
1333 else ((manifest_base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
1334 # If media_url is itself a f4m manifest do the recursive extraction
1335 # since bitrates in parent manifest (this one) and media_url manifest
1336 # may differ leading to inability to resolve the format by requested
1337 # bitrate in f4m downloader
1338 ext = determine_ext(manifest_url)
1340 f4m_formats = self._extract_f4m_formats(
1341 manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1342 transform_source=transform_source, fatal=fatal)
1343 # Sometimes stream-level manifest contains single media entry that
1344 # does not contain any quality metadata (e.g. http://matchtv.ru/#live-player).
1345 # At the same time parent's media entry in set-level manifest may
1346 # contain it. We will copy it from parent in such cases.
1347 if len(f4m_formats) == 1:
1350 'tbr': f.get('tbr') or tbr,
1351 'width': f.get('width') or width,
1352 'height': f.get('height') or height,
1353 'format_id': f.get('format_id') if not tbr else format_id,
1356 formats.extend(f4m_formats)
1359 formats.extend(self._extract_m3u8_formats(
1360 manifest_url, video_id, 'mp4', preference=preference,
1361 m3u8_id=m3u8_id, fatal=fatal))
1364 'format_id': format_id,
1365 'url': manifest_url,
1366 'manifest_url': manifest_url,
1367 'ext': 'flv' if bootstrap_info is not None else None,
1373 'preference': preference,
1377 def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, m3u8_id=None):
1379 'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
1383 'preference': preference - 100 if preference else -100,
1384 'resolution': 'multiple',
1385 'format_note': 'Quality selection URL',
1388 def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
1389 entry_protocol='m3u8', preference=None,
1390 m3u8_id=None, note=None, errnote=None,
1391 fatal=True, live=False):
1392 res = self._download_webpage_handle(
1394 note=note or 'Downloading m3u8 information',
1395 errnote=errnote or 'Failed to download m3u8 information',
1401 m3u8_doc, urlh = res
1402 m3u8_url = urlh.geturl()
1404 return self._parse_m3u8_formats(
1405 m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
1406 preference=preference, m3u8_id=m3u8_id, live=live)
1408 def _parse_m3u8_formats(self, m3u8_doc, m3u8_url, ext=None,
1409 entry_protocol='m3u8', preference=None,
1410 m3u8_id=None, live=False):
1411 if '#EXT-X-FAXS-CM:' in m3u8_doc: # Adobe Flash Access
1414 if re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc): # Apple FairPlay
1419 format_url = lambda u: (
1421 if re.match(r'^https?://', u)
1422 else compat_urlparse.urljoin(m3u8_url, u))
1425 # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
1426 # 2. https://github.com/rg3/youtube-dl/issues/12211
1428 # We should try extracting formats only from master playlists [1, 4.3.4],
1429 # i.e. playlists that describe available qualities. On the other hand
1430 # media playlists [1, 4.3.3] should be returned as is since they contain
1431 # just the media without qualities renditions.
1432 # Fortunately, master playlist can be easily distinguished from media
1433 # playlist based on particular tags availability. As of [1, 4.3.3, 4.3.4]
1434 # master playlist tags MUST NOT appear in a media playist and vice versa.
1435 # As of [1, 4.3.3.1] #EXT-X-TARGETDURATION tag is REQUIRED for every
1436 # media playlist and MUST NOT appear in master playlist thus we can
1437 # clearly detect media playlist with this criterion.
1439 if '#EXT-X-TARGETDURATION' in m3u8_doc: # media playlist, return as is
1442 'format_id': m3u8_id,
1444 'protocol': entry_protocol,
1445 'preference': preference,
1449 last_stream_inf = {}
1451 def extract_media(x_media_line):
1452 media = parse_m3u8_attributes(x_media_line)
1453 # As per [1, 4.3.4.1] TYPE, GROUP-ID and NAME are REQUIRED
1454 media_type, group_id, name = media.get('TYPE'), media.get('GROUP-ID'), media.get('NAME')
1455 if not (media_type and group_id and name):
1457 groups.setdefault(group_id, []).append(media)
1458 if media_type not in ('VIDEO', 'AUDIO'):
1460 media_url = media.get('URI')
1463 for v in (m3u8_id, group_id, name):
1467 'format_id': '-'.join(format_id),
1468 'url': format_url(media_url),
1469 'manifest_url': m3u8_url,
1470 'language': media.get('LANGUAGE'),
1472 'protocol': entry_protocol,
1473 'preference': preference,
1475 if media_type == 'AUDIO':
1476 f['vcodec'] = 'none'
1479 def build_stream_name():
1480 # Despite specification does not mention NAME attribute for
1481 # EXT-X-STREAM-INF tag it still sometimes may be present (see [1]
1482 # or vidio test in TestInfoExtractor.test_parse_m3u8_formats)
1483 # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
1484 stream_name = last_stream_inf.get('NAME')
1487 # If there is no NAME in EXT-X-STREAM-INF it will be obtained
1488 # from corresponding rendition group
1489 stream_group_id = last_stream_inf.get('VIDEO')
1490 if not stream_group_id:
1492 stream_group = groups.get(stream_group_id)
1493 if not stream_group:
1494 return stream_group_id
1495 rendition = stream_group[0]
1496 return rendition.get('NAME') or stream_group_id
1498 for line in m3u8_doc.splitlines():
1499 if line.startswith('#EXT-X-STREAM-INF:'):
1500 last_stream_inf = parse_m3u8_attributes(line)
1501 elif line.startswith('#EXT-X-MEDIA:'):
1503 elif line.startswith('#') or not line.strip():
1506 tbr = float_or_none(
1507 last_stream_inf.get('AVERAGE-BANDWIDTH') or
1508 last_stream_inf.get('BANDWIDTH'), scale=1000)
1511 format_id.append(m3u8_id)
1512 stream_name = build_stream_name()
1513 # Bandwidth of live streams may differ over time thus making
1514 # format_id unpredictable. So it's better to keep provided
1517 format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
1518 manifest_url = format_url(line.strip())
1520 'format_id': '-'.join(format_id),
1521 'url': manifest_url,
1522 'manifest_url': m3u8_url,
1525 'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
1526 'protocol': entry_protocol,
1527 'preference': preference,
1529 resolution = last_stream_inf.get('RESOLUTION')
1531 mobj = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', resolution)
1533 f['width'] = int(mobj.group('width'))
1534 f['height'] = int(mobj.group('height'))
1535 # Unified Streaming Platform
1537 r'audio.*?(?:%3D|=)(\d+)(?:-video.*?(?:%3D|=)(\d+))?', f['url'])
1539 abr, vbr = mobj.groups()
1540 abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
1545 codecs = parse_codecs(last_stream_inf.get('CODECS'))
1547 audio_group_id = last_stream_inf.get('AUDIO')
1548 # As per [1, 4.3.4.1.1] any EXT-X-STREAM-INF tag which
1549 # references a rendition group MUST have a CODECS attribute.
1550 # However, this is not always respected, for example, [2]
1551 # contains EXT-X-STREAM-INF tag which references AUDIO
1552 # rendition group but does not have CODECS and despite
1553 # referencing audio group an audio group, it represents
1554 # a complete (with audio and video) format. So, for such cases
1555 # we will ignore references to rendition groups and treat them
1556 # as complete formats.
1557 if audio_group_id and codecs and f.get('vcodec') != 'none':
1558 audio_group = groups.get(audio_group_id)
1559 if audio_group and audio_group[0].get('URI'):
1560 # TODO: update acodec for audio only formats with
1562 f['acodec'] = 'none'
1564 last_stream_inf = {}
1568 def _xpath_ns(path, namespace=None):
1572 for c in path.split('/'):
1573 if not c or c == '.':
1576 out.append('{%s}%s' % (namespace, c))
1577 return '/'.join(out)
1579 def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
1580 smil = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
1586 namespace = self._parse_smil_namespace(smil)
1588 return self._parse_smil_formats(
1589 smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1591 def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
1592 smil = self._download_smil(smil_url, video_id, fatal=fatal)
1595 return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
1597 def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
1598 return self._download_xml(
1599 smil_url, video_id, 'Downloading SMIL file',
1600 'Unable to download SMIL file', fatal=fatal, transform_source=transform_source)
1602 def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
1603 namespace = self._parse_smil_namespace(smil)
1605 formats = self._parse_smil_formats(
1606 smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1607 subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
1609 video_id = os.path.splitext(url_basename(smil_url))[0]
1613 for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1614 name = meta.attrib.get('name')
1615 content = meta.attrib.get('content')
1616 if not name or not content:
1618 if not title and name == 'title':
1620 elif not description and name in ('description', 'abstract'):
1621 description = content
1622 elif not upload_date and name == 'date':
1623 upload_date = unified_strdate(content)
1626 'id': image.get('type'),
1627 'url': image.get('src'),
1628 'width': int_or_none(image.get('width')),
1629 'height': int_or_none(image.get('height')),
1630 } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
1634 'title': title or video_id,
1635 'description': description,
1636 'upload_date': upload_date,
1637 'thumbnails': thumbnails,
1639 'subtitles': subtitles,
1642 def _parse_smil_namespace(self, smil):
1643 return self._search_regex(
1644 r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
1646 def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
1648 for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1649 b = meta.get('base') or meta.get('httpBase')
1660 media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
1661 for medium in media:
1662 src = medium.get('src')
1663 if not src or src in srcs:
1667 bitrate = float_or_none(medium.get('system-bitrate') or medium.get('systemBitrate'), 1000)
1668 filesize = int_or_none(medium.get('size') or medium.get('fileSize'))
1669 width = int_or_none(medium.get('width'))
1670 height = int_or_none(medium.get('height'))
1671 proto = medium.get('proto')
1672 ext = medium.get('ext')
1673 src_ext = determine_ext(src)
1674 streamer = medium.get('streamer') or base
1676 if proto == 'rtmp' or streamer.startswith('rtmp'):
1682 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
1684 'filesize': filesize,
1688 if transform_rtmp_url:
1689 streamer, src = transform_rtmp_url(streamer, src)
1690 formats[-1].update({
1696 src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
1697 src_url = src_url.strip()
1699 if proto == 'm3u8' or src_ext == 'm3u8':
1700 m3u8_formats = self._extract_m3u8_formats(
1701 src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
1702 if len(m3u8_formats) == 1:
1704 m3u8_formats[0].update({
1705 'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
1710 formats.extend(m3u8_formats)
1713 if src_ext == 'f4m':
1718 'plugin': 'flowplayer-3.2.0.1',
1720 f4m_url += '&' if '?' in f4m_url else '?'
1721 f4m_url += compat_urllib_parse_urlencode(f4m_params)
1722 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
1725 if src_url.startswith('http') and self._is_valid_url(src, video_id):
1729 'ext': ext or src_ext or 'flv',
1730 'format_id': 'http-%d' % (bitrate or http_count),
1732 'filesize': filesize,
1740 def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
1743 for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
1744 src = textstream.get('src')
1745 if not src or src in urls:
1748 ext = textstream.get('ext') or mimetype2ext(textstream.get('type')) or determine_ext(src)
1749 lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
1750 subtitles.setdefault(lang, []).append({
1756 def _extract_xspf_playlist(self, xspf_url, playlist_id, fatal=True):
1757 xspf = self._download_xml(
1758 xspf_url, playlist_id, 'Downloading xpsf playlist',
1759 'Unable to download xspf manifest', fatal=fatal)
1762 return self._parse_xspf(
1763 xspf, playlist_id, xspf_url=xspf_url,
1764 xspf_base_url=base_url(xspf_url))
1766 def _parse_xspf(self, xspf_doc, playlist_id, xspf_url=None, xspf_base_url=None):
1768 'xspf': 'http://xspf.org/ns/0/',
1769 's1': 'http://static.streamone.nl/player/ns/0',
1773 for track in xspf_doc.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
1775 track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
1776 description = xpath_text(
1777 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
1778 thumbnail = xpath_text(
1779 track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
1780 duration = float_or_none(
1781 xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
1784 for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP)):
1785 format_url = urljoin(xspf_base_url, location.text)
1790 'manifest_url': xspf_url,
1791 'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
1792 'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
1793 'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
1795 self._sort_formats(formats)
1800 'description': description,
1801 'thumbnail': thumbnail,
1802 'duration': duration,
1807 def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}):
1808 res = self._download_xml_handle(
1810 note=note or 'Downloading MPD manifest',
1811 errnote=errnote or 'Failed to download MPD manifest',
1816 mpd_base_url = base_url(urlh.geturl())
1818 return self._parse_mpd_formats(
1819 mpd_doc, mpd_id=mpd_id, mpd_base_url=mpd_base_url,
1820 formats_dict=formats_dict, mpd_url=mpd_url)
1822 def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}, mpd_url=None):
1824 Parse formats from MPD manifest.
1826 1. MPEG-DASH Standard, ISO/IEC 23009-1:2014(E),
1827 http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip
1828 2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
1830 if mpd_doc.get('type') == 'dynamic':
1833 namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
1836 return self._xpath_ns(path, namespace)
1838 def is_drm_protected(element):
1839 return element.find(_add_ns('ContentProtection')) is not None
1841 def extract_multisegment_info(element, ms_parent_info):
1842 ms_info = ms_parent_info.copy()
1844 # As per [1, 5.3.9.2.2] SegmentList and SegmentTemplate share some
1845 # common attributes and elements. We will only extract relevant
1847 def extract_common(source):
1848 segment_timeline = source.find(_add_ns('SegmentTimeline'))
1849 if segment_timeline is not None:
1850 s_e = segment_timeline.findall(_add_ns('S'))
1852 ms_info['total_number'] = 0
1855 r = int(s.get('r', 0))
1856 ms_info['total_number'] += 1 + r
1857 ms_info['s'].append({
1858 't': int(s.get('t', 0)),
1859 # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
1860 'd': int(s.attrib['d']),
1863 start_number = source.get('startNumber')
1865 ms_info['start_number'] = int(start_number)
1866 timescale = source.get('timescale')
1868 ms_info['timescale'] = int(timescale)
1869 segment_duration = source.get('duration')
1870 if segment_duration:
1871 ms_info['segment_duration'] = float(segment_duration)
1873 def extract_Initialization(source):
1874 initialization = source.find(_add_ns('Initialization'))
1875 if initialization is not None:
1876 ms_info['initialization_url'] = initialization.attrib['sourceURL']
1878 segment_list = element.find(_add_ns('SegmentList'))
1879 if segment_list is not None:
1880 extract_common(segment_list)
1881 extract_Initialization(segment_list)
1882 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
1884 ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
1886 segment_template = element.find(_add_ns('SegmentTemplate'))
1887 if segment_template is not None:
1888 extract_common(segment_template)
1889 media = segment_template.get('media')
1891 ms_info['media'] = media
1892 initialization = segment_template.get('initialization')
1894 ms_info['initialization'] = initialization
1896 extract_Initialization(segment_template)
1899 mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
1901 for period in mpd_doc.findall(_add_ns('Period')):
1902 period_duration = parse_duration(period.get('duration')) or mpd_duration
1903 period_ms_info = extract_multisegment_info(period, {
1907 for adaptation_set in period.findall(_add_ns('AdaptationSet')):
1908 if is_drm_protected(adaptation_set):
1910 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
1911 for representation in adaptation_set.findall(_add_ns('Representation')):
1912 if is_drm_protected(representation):
1914 representation_attrib = adaptation_set.attrib.copy()
1915 representation_attrib.update(representation.attrib)
1916 # According to [1, 5.3.7.2, Table 9, page 41], @mimeType is mandatory
1917 mime_type = representation_attrib['mimeType']
1918 content_type = mime_type.split('/')[0]
1919 if content_type == 'text':
1920 # TODO implement WebVTT downloading
1922 elif content_type in ('video', 'audio'):
1924 for element in (representation, adaptation_set, period, mpd_doc):
1925 base_url_e = element.find(_add_ns('BaseURL'))
1926 if base_url_e is not None:
1927 base_url = base_url_e.text + base_url
1928 if re.match(r'^https?://', base_url):
1930 if mpd_base_url and not re.match(r'^https?://', base_url):
1931 if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
1933 base_url = mpd_base_url + base_url
1934 representation_id = representation_attrib.get('id')
1935 lang = representation_attrib.get('lang')
1936 url_el = representation.find(_add_ns('BaseURL'))
1937 filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
1938 bandwidth = int_or_none(representation_attrib.get('bandwidth'))
1940 'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
1942 'manifest_url': mpd_url,
1943 'ext': mimetype2ext(mime_type),
1944 'width': int_or_none(representation_attrib.get('width')),
1945 'height': int_or_none(representation_attrib.get('height')),
1946 'tbr': float_or_none(bandwidth, 1000),
1947 'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
1948 'fps': int_or_none(representation_attrib.get('frameRate')),
1949 'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
1950 'format_note': 'DASH %s' % content_type,
1951 'filesize': filesize,
1952 'container': mimetype2ext(mime_type) + '_dash',
1954 f.update(parse_codecs(representation_attrib.get('codecs')))
1955 representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
1957 def prepare_template(template_name, identifiers):
1958 t = representation_ms_info[template_name]
1959 t = t.replace('$RepresentationID$', representation_id)
1960 t = re.sub(r'\$(%s)\$' % '|'.join(identifiers), r'%(\1)d', t)
1961 t = re.sub(r'\$(%s)%%([^$]+)\$' % '|'.join(identifiers), r'%(\1)\2', t)
1962 t.replace('$$', '$')
1965 # @initialization is a regular template like @media one
1966 # so it should be handled just the same way (see
1967 # https://github.com/rg3/youtube-dl/issues/11605)
1968 if 'initialization' in representation_ms_info:
1969 initialization_template = prepare_template(
1971 # As per [1, 5.3.9.4.2, Table 15, page 54] $Number$ and
1972 # $Time$ shall not be included for @initialization thus
1973 # only $Bandwidth$ remains
1975 representation_ms_info['initialization_url'] = initialization_template % {
1976 'Bandwidth': bandwidth,
1979 def location_key(location):
1980 return 'url' if re.match(r'^https?://', location) else 'path'
1982 if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
1984 media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
1985 media_location_key = location_key(media_template)
1987 # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
1988 # can't be used at the same time
1989 if '%(Number' in media_template and 's' not in representation_ms_info:
1990 segment_duration = None
1991 if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
1992 segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
1993 representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
1994 representation_ms_info['fragments'] = [{
1995 media_location_key: media_template % {
1996 'Number': segment_number,
1997 'Bandwidth': bandwidth,
1999 'duration': segment_duration,
2000 } for segment_number in range(
2001 representation_ms_info['start_number'],
2002 representation_ms_info['total_number'] + representation_ms_info['start_number'])]
2004 # $Number*$ or $Time$ in media template with S list available
2005 # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
2006 # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
2007 representation_ms_info['fragments'] = []
2010 segment_number = representation_ms_info['start_number']
2012 def add_segment_url():
2013 segment_url = media_template % {
2014 'Time': segment_time,
2015 'Bandwidth': bandwidth,
2016 'Number': segment_number,
2018 representation_ms_info['fragments'].append({
2019 media_location_key: segment_url,
2020 'duration': float_or_none(segment_d, representation_ms_info['timescale']),
2023 for num, s in enumerate(representation_ms_info['s']):
2024 segment_time = s.get('t') or segment_time
2028 for r in range(s.get('r', 0)):
2029 segment_time += segment_d
2032 segment_time += segment_d
2033 elif 'segment_urls' in representation_ms_info and 's' in representation_ms_info:
2035 # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
2036 # or any YouTube dashsegments video
2039 timescale = representation_ms_info['timescale']
2040 for s in representation_ms_info['s']:
2041 duration = float_or_none(s['d'], timescale)
2042 for r in range(s.get('r', 0) + 1):
2043 segment_uri = representation_ms_info['segment_urls'][segment_index]
2045 location_key(segment_uri): segment_uri,
2046 'duration': duration,
2049 representation_ms_info['fragments'] = fragments
2050 elif 'segment_urls' in representation_ms_info:
2051 # Segment URLs with no SegmentTimeline
2052 # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
2053 # https://github.com/rg3/youtube-dl/pull/14844
2055 segment_duration = float_or_none(
2056 representation_ms_info['segment_duration'],
2057 representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
2058 for segment_url in representation_ms_info['segment_urls']:
2060 location_key(segment_url): segment_url,
2062 if segment_duration:
2063 fragment['duration'] = segment_duration
2064 fragments.append(fragment)
2065 representation_ms_info['fragments'] = fragments
2066 # NB: MPD manifest may contain direct URLs to unfragmented media.
2067 # No fragments key is present in this case.
2068 if 'fragments' in representation_ms_info:
2070 'fragment_base_url': base_url,
2072 'protocol': 'http_dash_segments',
2074 if 'initialization_url' in representation_ms_info:
2075 initialization_url = representation_ms_info['initialization_url']
2076 if not f.get('url'):
2077 f['url'] = initialization_url
2078 f['fragments'].append({location_key(initialization_url): initialization_url})
2079 f['fragments'].extend(representation_ms_info['fragments'])
2080 # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
2081 # is not necessarily unique within a Period thus formats with
2082 # the same `format_id` are quite possible. There are numerous examples
2083 # of such manifests (see https://github.com/rg3/youtube-dl/issues/15111,
2084 # https://github.com/rg3/youtube-dl/issues/13919)
2085 full_info = formats_dict.get(representation_id, {}).copy()
2087 formats.append(full_info)
2089 self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
2092 def _extract_ism_formats(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True):
2093 res = self._download_xml_handle(
2095 note=note or 'Downloading ISM manifest',
2096 errnote=errnote or 'Failed to download ISM manifest',
2102 return self._parse_ism_formats(ism_doc, urlh.geturl(), ism_id)
2104 def _parse_ism_formats(self, ism_doc, ism_url, ism_id=None):
2106 Parse formats from ISM manifest.
2108 1. [MS-SSTR]: Smooth Streaming Protocol,
2109 https://msdn.microsoft.com/en-us/library/ff469518.aspx
2111 if ism_doc.get('IsLive') == 'TRUE' or ism_doc.find('Protection') is not None:
2114 duration = int(ism_doc.attrib['Duration'])
2115 timescale = int_or_none(ism_doc.get('TimeScale')) or 10000000
2118 for stream in ism_doc.findall('StreamIndex'):
2119 stream_type = stream.get('Type')
2120 if stream_type not in ('video', 'audio'):
2122 url_pattern = stream.attrib['Url']
2123 stream_timescale = int_or_none(stream.get('TimeScale')) or timescale
2124 stream_name = stream.get('Name')
2125 for track in stream.findall('QualityLevel'):
2126 fourcc = track.get('FourCC', 'AACL' if track.get('AudioTag') == '255' else None)
2127 # TODO: add support for WVC1 and WMAP
2128 if fourcc not in ('H264', 'AVC1', 'AACL'):
2129 self.report_warning('%s is not a supported codec' % fourcc)
2131 tbr = int(track.attrib['Bitrate']) // 1000
2132 # [1] does not mention Width and Height attributes. However,
2133 # they're often present while MaxWidth and MaxHeight are
2134 # missing, so should be used as fallbacks
2135 width = int_or_none(track.get('MaxWidth') or track.get('Width'))
2136 height = int_or_none(track.get('MaxHeight') or track.get('Height'))
2137 sampling_rate = int_or_none(track.get('SamplingRate'))
2139 track_url_pattern = re.sub(r'{[Bb]itrate}', track.attrib['Bitrate'], url_pattern)
2140 track_url_pattern = compat_urlparse.urljoin(ism_url, track_url_pattern)
2146 stream_fragments = stream.findall('c')
2147 for stream_fragment_index, stream_fragment in enumerate(stream_fragments):
2148 fragment_ctx['time'] = int_or_none(stream_fragment.get('t')) or fragment_ctx['time']
2149 fragment_repeat = int_or_none(stream_fragment.get('r')) or 1
2150 fragment_ctx['duration'] = int_or_none(stream_fragment.get('d'))
2151 if not fragment_ctx['duration']:
2153 next_fragment_time = int(stream_fragment[stream_fragment_index + 1].attrib['t'])
2155 next_fragment_time = duration
2156 fragment_ctx['duration'] = (next_fragment_time - fragment_ctx['time']) / fragment_repeat
2157 for _ in range(fragment_repeat):
2159 'url': re.sub(r'{start[ _]time}', compat_str(fragment_ctx['time']), track_url_pattern),
2160 'duration': fragment_ctx['duration'] / stream_timescale,
2162 fragment_ctx['time'] += fragment_ctx['duration']
2166 format_id.append(ism_id)
2168 format_id.append(stream_name)
2169 format_id.append(compat_str(tbr))
2172 'format_id': '-'.join(format_id),
2174 'manifest_url': ism_url,
2175 'ext': 'ismv' if stream_type == 'video' else 'isma',
2179 'asr': sampling_rate,
2180 'vcodec': 'none' if stream_type == 'audio' else fourcc,
2181 'acodec': 'none' if stream_type == 'video' else fourcc,
2183 'fragments': fragments,
2184 '_download_params': {
2185 'duration': duration,
2186 'timescale': stream_timescale,
2187 'width': width or 0,
2188 'height': height or 0,
2190 'codec_private_data': track.get('CodecPrivateData'),
2191 'sampling_rate': sampling_rate,
2192 'channels': int_or_none(track.get('Channels', 2)),
2193 'bits_per_sample': int_or_none(track.get('BitsPerSample', 16)),
2194 'nal_unit_length_field': int_or_none(track.get('NALUnitLengthField', 4)),
2199 def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8', mpd_id=None, preference=None):
2200 def absolute_url(item_url):
2201 return urljoin(base_url, item_url)
2203 def parse_content_type(content_type):
2204 if not content_type:
2206 ctr = re.search(r'(?P<mimetype>[^/]+/[^;]+)(?:;\s*codecs="?(?P<codecs>[^"]+))?', content_type)
2208 mimetype, codecs = ctr.groups()
2209 f = parse_codecs(codecs)
2210 f['ext'] = mimetype2ext(mimetype)
2214 def _media_formats(src, cur_media_type, type_info={}):
2215 full_url = absolute_url(src)
2216 ext = type_info.get('ext') or determine_ext(full_url)
2218 is_plain_url = False
2219 formats = self._extract_m3u8_formats(
2220 full_url, video_id, ext='mp4',
2221 entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id,
2222 preference=preference, fatal=False)
2224 is_plain_url = False
2225 formats = self._extract_mpd_formats(
2226 full_url, video_id, mpd_id=mpd_id, fatal=False)
2231 'vcodec': 'none' if cur_media_type == 'audio' else None,
2233 return is_plain_url, formats
2236 # amp-video and amp-audio are very similar to their HTML5 counterparts
2237 # so we wll include them right here (see
2238 # https://www.ampproject.org/docs/reference/components/amp-video)
2239 media_tags = [(media_tag, media_type, '')
2240 for media_tag, media_type
2241 in re.findall(r'(?s)(<(?:amp-)?(video|audio)[^>]*/>)', webpage)]
2242 media_tags.extend(re.findall(
2243 # We only allow video|audio followed by a whitespace or '>'.
2244 # Allowing more characters may end up in significant slow down (see
2245 # https://github.com/rg3/youtube-dl/issues/11979, example URL:
2246 # http://www.porntrex.com/maps/videositemap.xml).
2247 r'(?s)(<(?P<tag>(?:amp-)?(?:video|audio))(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
2248 for media_tag, media_type, media_content in media_tags:
2253 media_attributes = extract_attributes(media_tag)
2254 src = media_attributes.get('src')
2256 _, formats = _media_formats(src, media_type)
2257 media_info['formats'].extend(formats)
2258 media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
2260 for source_tag in re.findall(r'<source[^>]+>', media_content):
2261 source_attributes = extract_attributes(source_tag)
2262 src = source_attributes.get('src')
2265 f = parse_content_type(source_attributes.get('type'))
2266 is_plain_url, formats = _media_formats(src, media_type, f)
2268 # res attribute is not standard but seen several times
2271 'height': int_or_none(source_attributes.get('res')),
2272 'format_id': source_attributes.get('label'),
2274 f.update(formats[0])
2275 media_info['formats'].append(f)
2277 media_info['formats'].extend(formats)
2278 for track_tag in re.findall(r'<track[^>]+>', media_content):
2279 track_attributes = extract_attributes(track_tag)
2280 kind = track_attributes.get('kind')
2281 if not kind or kind in ('subtitles', 'captions'):
2282 src = track_attributes.get('src')
2285 lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')
2286 media_info['subtitles'].setdefault(lang, []).append({
2287 'url': absolute_url(src),
2289 if media_info['formats'] or media_info['subtitles']:
2290 entries.append(media_info)
2293 def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
2295 hdcore_sign = 'hdcore=3.7.0'
2296 f4m_url = re.sub(r'(https?://[^/]+)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
2297 hds_host = hosts.get('hds')
2299 f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)
2300 if 'hdcore=' not in f4m_url:
2301 f4m_url += ('&' if '?' in f4m_url else '?') + hdcore_sign
2302 f4m_formats = self._extract_f4m_formats(
2303 f4m_url, video_id, f4m_id='hds', fatal=False)
2304 for entry in f4m_formats:
2305 entry.update({'extra_param_to_segment_url': hdcore_sign})
2306 formats.extend(f4m_formats)
2307 m3u8_url = re.sub(r'(https?://[^/]+)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
2308 hls_host = hosts.get('hls')
2310 m3u8_url = re.sub(r'(https?://)[^/]+', r'\1' + hls_host, m3u8_url)
2311 formats.extend(self._extract_m3u8_formats(
2312 m3u8_url, video_id, 'mp4', 'm3u8_native',
2313 m3u8_id='hls', fatal=False))
2316 def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
2317 query = compat_urlparse.urlparse(url).query
2318 url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
2320 r'(?:(?:http|rtmp|rtsp)(?P<s>s)?:)?(?P<url>//[^?]+)', url)
2321 url_base = mobj.group('url')
2322 http_base_url = '%s%s:%s' % ('http', mobj.group('s') or '', url_base)
2325 def manifest_url(manifest):
2326 m_url = '%s/%s' % (http_base_url, manifest)
2328 m_url += '?%s' % query
2331 if 'm3u8' not in skip_protocols:
2332 formats.extend(self._extract_m3u8_formats(
2333 manifest_url('playlist.m3u8'), video_id, 'mp4',
2334 m3u8_entry_protocol, m3u8_id='hls', fatal=False))
2335 if 'f4m' not in skip_protocols:
2336 formats.extend(self._extract_f4m_formats(
2337 manifest_url('manifest.f4m'),
2338 video_id, f4m_id='hds', fatal=False))
2339 if 'dash' not in skip_protocols:
2340 formats.extend(self._extract_mpd_formats(
2341 manifest_url('manifest.mpd'),
2342 video_id, mpd_id='dash', fatal=False))
2343 if re.search(r'(?:/smil:|\.smil)', url_base):
2344 if 'smil' not in skip_protocols:
2345 rtmp_formats = self._extract_smil_formats(
2346 manifest_url('jwplayer.smil'),
2347 video_id, fatal=False)
2348 for rtmp_format in rtmp_formats:
2349 rtsp_format = rtmp_format.copy()
2350 rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
2351 del rtsp_format['play_path']
2352 del rtsp_format['ext']
2353 rtsp_format.update({
2354 'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
2355 'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
2358 formats.extend([rtmp_format, rtsp_format])
2360 for protocol in ('rtmp', 'rtsp'):
2361 if protocol not in skip_protocols:
2363 'url': '%s:%s' % (protocol, url_base),
2364 'format_id': protocol,
2365 'protocol': protocol,
2369 def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
2371 r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
2375 jwplayer_data = self._parse_json(mobj.group('options'),
2377 transform_source=transform_source)
2378 except ExtractorError:
2381 if isinstance(jwplayer_data, dict):
2382 return jwplayer_data
2384 def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
2385 jwplayer_data = self._find_jwplayer_data(
2386 webpage, video_id, transform_source=js_to_json)
2387 return self._parse_jwplayer_data(
2388 jwplayer_data, video_id, *args, **kwargs)
2390 def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
2391 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2392 # JWPlayer backward compatibility: flattened playlists
2393 # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
2394 if 'playlist' not in jwplayer_data:
2395 jwplayer_data = {'playlist': [jwplayer_data]}
2399 # JWPlayer backward compatibility: single playlist item
2400 # https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
2401 if not isinstance(jwplayer_data['playlist'], list):
2402 jwplayer_data['playlist'] = [jwplayer_data['playlist']]
2404 for video_data in jwplayer_data['playlist']:
2405 # JWPlayer backward compatibility: flattened sources
2406 # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
2407 if 'sources' not in video_data:
2408 video_data['sources'] = [video_data]
2410 this_video_id = video_id or video_data['mediaid']
2412 formats = self._parse_jwplayer_formats(
2413 video_data['sources'], video_id=this_video_id, m3u8_id=m3u8_id,
2414 mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
2417 tracks = video_data.get('tracks')
2418 if tracks and isinstance(tracks, list):
2419 for track in tracks:
2420 if not isinstance(track, dict):
2422 track_kind = track.get('kind')
2423 if not track_kind or not isinstance(track_kind, compat_str):
2425 if track_kind.lower() not in ('captions', 'subtitles'):
2427 track_url = urljoin(base_url, track.get('file'))
2430 subtitles.setdefault(track.get('label') or 'en', []).append({
2431 'url': self._proto_relative_url(track_url)
2435 'id': this_video_id,
2436 'title': unescapeHTML(video_data['title'] if require_title else video_data.get('title')),
2437 'description': video_data.get('description'),
2438 'thumbnail': self._proto_relative_url(video_data.get('image')),
2439 'timestamp': int_or_none(video_data.get('pubdate')),
2440 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
2441 'subtitles': subtitles,
2443 # https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
2444 if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
2446 '_type': 'url_transparent',
2447 'url': formats[0]['url'],
2450 self._sort_formats(formats)
2451 entry['formats'] = formats
2452 entries.append(entry)
2453 if len(entries) == 1:
2456 return self.playlist_result(entries)
2458 def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
2459 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2462 for source in jwplayer_sources_data:
2463 if not isinstance(source, dict):
2465 source_url = self._proto_relative_url(source.get('file'))
2469 source_url = compat_urlparse.urljoin(base_url, source_url)
2470 if source_url in urls:
2472 urls.append(source_url)
2473 source_type = source.get('type') or ''
2474 ext = mimetype2ext(source_type) or determine_ext(source_url)
2475 if source_type == 'hls' or ext == 'm3u8':
2476 formats.extend(self._extract_m3u8_formats(
2477 source_url, video_id, 'mp4', entry_protocol='m3u8_native',
2478 m3u8_id=m3u8_id, fatal=False))
2479 elif source_type == 'dash' or ext == 'mpd':
2480 formats.extend(self._extract_mpd_formats(
2481 source_url, video_id, mpd_id=mpd_id, fatal=False))
2483 formats.extend(self._extract_smil_formats(
2484 source_url, video_id, fatal=False))
2485 # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
2486 elif source_type.startswith('audio') or ext in (
2487 'oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
2494 height = int_or_none(source.get('height'))
2496 # Often no height is provided but there is a label in
2497 # format like "1080p", "720p SD", or 1080.
2498 height = int_or_none(self._search_regex(
2499 r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
2500 'height', default=None))
2503 'width': int_or_none(source.get('width')),
2505 'tbr': int_or_none(source.get('bitrate')),
2508 if source_url.startswith('rtmp'):
2509 a_format['ext'] = 'flv'
2510 # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
2511 # of jwplayer.flash.swf
2512 rtmp_url_parts = re.split(
2513 r'((?:mp4|mp3|flv):)', source_url, 1)
2514 if len(rtmp_url_parts) == 3:
2515 rtmp_url, prefix, play_path = rtmp_url_parts
2518 'play_path': prefix + play_path,
2521 a_format.update(rtmp_params)
2522 formats.append(a_format)
2525 def _live_title(self, name):
2526 """ Generate the title for a live video """
2527 now = datetime.datetime.now()
2528 now_str = now.strftime('%Y-%m-%d %H:%M')
2529 return name + ' ' + now_str
2531 def _int(self, v, name, fatal=False, **kwargs):
2532 res = int_or_none(v, **kwargs)
2533 if 'get_attr' in kwargs:
2534 print(getattr(v, kwargs['get_attr']))
2536 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2538 raise ExtractorError(msg)
2540 self._downloader.report_warning(msg)
2543 def _float(self, v, name, fatal=False, **kwargs):
2544 res = float_or_none(v, **kwargs)
2546 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2548 raise ExtractorError(msg)
2550 self._downloader.report_warning(msg)
2553 def _set_cookie(self, domain, name, value, expire_time=None, port=None,
2554 path='/', secure=False, discard=False, rest={}, **kwargs):
2555 cookie = compat_cookiejar.Cookie(
2556 0, name, value, port, port is not None, domain, True,
2557 domain.startswith('.'), path, True, secure, expire_time,
2558 discard, None, None, rest)
2559 self._downloader.cookiejar.set_cookie(cookie)
2561 def _get_cookies(self, url):
2562 """ Return a compat_cookies.SimpleCookie with the cookies for the url """
2563 req = sanitized_Request(url)
2564 self._downloader.cookiejar.add_cookie_header(req)
2565 return compat_cookies.SimpleCookie(req.get_header('Cookie'))
2567 def get_testcases(self, include_onlymatching=False):
2568 t = getattr(self, '_TEST', None)
2570 assert not hasattr(self, '_TESTS'), \
2571 '%s has _TEST and _TESTS' % type(self).__name__
2574 tests = getattr(self, '_TESTS', [])
2576 if not include_onlymatching and t.get('only_matching', False):
2578 t['name'] = type(self).__name__[:-len('IE')]
2581 def is_suitable(self, age_limit):
2582 """ Test whether the extractor is generally suitable for the given
2583 age limit (i.e. pornographic sites are not, all others usually are) """
2585 any_restricted = False
2586 for tc in self.get_testcases(include_onlymatching=False):
2587 if tc.get('playlist', []):
2588 tc = tc['playlist'][0]
2589 is_restricted = age_restricted(
2590 tc.get('info_dict', {}).get('age_limit'), age_limit)
2591 if not is_restricted:
2593 any_restricted = any_restricted or is_restricted
2594 return not any_restricted
2596 def extract_subtitles(self, *args, **kwargs):
2597 if (self._downloader.params.get('writesubtitles', False) or
2598 self._downloader.params.get('listsubtitles')):
2599 return self._get_subtitles(*args, **kwargs)
2602 def _get_subtitles(self, *args, **kwargs):
2603 raise NotImplementedError('This method must be implemented by subclasses')
2606 def _merge_subtitle_items(subtitle_list1, subtitle_list2):
2607 """ Merge subtitle items for one language. Items with duplicated URLs
2608 will be dropped. """
2609 list1_urls = set([item['url'] for item in subtitle_list1])
2610 ret = list(subtitle_list1)
2611 ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
2615 def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
2616 """ Merge two subtitle dictionaries, language by language. """
2617 ret = dict(subtitle_dict1)
2618 for lang in subtitle_dict2:
2619 ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
2622 def extract_automatic_captions(self, *args, **kwargs):
2623 if (self._downloader.params.get('writeautomaticsub', False) or
2624 self._downloader.params.get('listsubtitles')):
2625 return self._get_automatic_captions(*args, **kwargs)
2628 def _get_automatic_captions(self, *args, **kwargs):
2629 raise NotImplementedError('This method must be implemented by subclasses')
2631 def mark_watched(self, *args, **kwargs):
2632 if (self._downloader.params.get('mark_watched', False) and
2633 (self._get_login_info()[0] is not None or
2634 self._downloader.params.get('cookiefile') is not None)):
2635 self._mark_watched(*args, **kwargs)
2637 def _mark_watched(self, *args, **kwargs):
2638 raise NotImplementedError('This method must be implemented by subclasses')
2640 def geo_verification_headers(self):
2642 geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
2643 if geo_verification_proxy:
2644 headers['Ytdl-request-proxy'] = geo_verification_proxy
2647 def _generic_id(self, url):
2648 return compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
2650 def _generic_title(self, url):
2651 return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
2654 class SearchInfoExtractor(InfoExtractor):
2656 Base class for paged search queries extractors.
2657 They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
2658 Instances should define _SEARCH_KEY and _MAX_RESULTS.
2662 def _make_valid_url(cls):
2663 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
2666 def suitable(cls, url):
2667 return re.match(cls._make_valid_url(), url) is not None
2669 def _real_extract(self, query):
2670 mobj = re.match(self._make_valid_url(), query)
2672 raise ExtractorError('Invalid search query "%s"' % query)
2674 prefix = mobj.group('prefix')
2675 query = mobj.group('query')
2677 return self._get_n_results(query, 1)
2678 elif prefix == 'all':
2679 return self._get_n_results(query, self._MAX_RESULTS)
2683 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
2684 elif n > self._MAX_RESULTS:
2685 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
2686 n = self._MAX_RESULTS
2687 return self._get_n_results(query, n)
2689 def _get_n_results(self, query, n):
2690 """Get a specified number of results for a query"""
2691 raise NotImplementedError('This method must be implemented by subclasses')
2694 def SEARCH_KEY(self):
2695 return self._SEARCH_KEY