X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fcommon.py;h=144d8c6b6dbfea83de4c319db8f238e34ca42730;hb=d800609c62703e4e6edd2891a8432306462e4db3;hp=65520744799013fbaa756d171cb10bff8a3e693e;hpb=126d7701b0f88c33b5fbda6fd07ef056b96c9367;p=youtube-dl diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 655207447..144d8c6b6 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -10,6 +10,7 @@ import re import socket import sys import time +import math from ..compat import ( compat_cookiejar, @@ -34,6 +35,7 @@ from ..utils import ( fix_xml_ampersands, float_or_none, int_or_none, + parse_iso8601, RegexNotFoundError, sanitize_filename, sanitized_Request, @@ -43,6 +45,7 @@ from ..utils import ( xpath_text, xpath_with_ns, determine_protocol, + parse_duration, ) @@ -108,8 +111,9 @@ class InfoExtractor(object): -2 or smaller for less than default. < -1000 to hide the format (if there is another one which is strictly better) - * language_preference Is this in the correct requested - language? + * language Language code, e.g. "de" or "en-US". + * language_preference Is this in the language mentioned in + the URL? 10 if it's what the URL is about, -1 for default (don't know), -10 otherwise, other values reserved for now. @@ -200,6 +204,26 @@ class InfoExtractor(object): end_time: Time in seconds where the reproduction should end, as specified in the URL. + The following fields should only be used when the video belongs to some logical + chapter or section: + + chapter: Name or title of the chapter the video belongs to. + chapter_number: Number of the chapter the video belongs to, as an integer. + chapter_id: Id of the chapter the video belongs to, as a unicode string. + + The following fields should only be used when the video is an episode of some + series or programme: + + series: Title of the series or programme the video episode belongs to. + season: Title of the season the video episode belongs to. + season_number: Number of the season the video episode belongs to, as an integer. + season_id: Id of the season the video episode belongs to, as a unicode string. + episode: Title of the video episode. Unlike mandatory video title field, + this field should denote the exact title of the video episode + without any kind of decoration. + episode_number: Number of the video episode within a season, as an integer. + episode_id: Id of the video episode, as a unicode string. + Unless mentioned otherwise, the fields should be Unicode strings. Unless mentioned otherwise, None is equivalent to absence of information. @@ -292,9 +316,9 @@ class InfoExtractor(object): except ExtractorError: raise except compat_http_client.IncompleteRead as e: - raise ExtractorError('A network error has occured.', cause=e, expected=True) + raise ExtractorError('A network error has occurred.', cause=e, expected=True) except (KeyError, StopIteration) as e: - raise ExtractorError('An extractor error has occured.', cause=e) + raise ExtractorError('An extractor error has occurred.', cause=e) def set_downloader(self, downloader): """Sets the downloader for this IE.""" @@ -612,7 +636,7 @@ class InfoExtractor(object): downloader_params = self._downloader.params # Attempt to use provided username and password or .netrc data - if downloader_params.get('username', None) is not None: + if downloader_params.get('username') is not None: username = downloader_params['username'] password = downloader_params['password'] elif downloader_params.get('usenetrc', False): @@ -639,7 +663,7 @@ class InfoExtractor(object): return None downloader_params = self._downloader.params - if downloader_params.get('twofactor', None) is not None: + if downloader_params.get('twofactor') is not None: return downloader_params['twofactor'] return compat_getpass('Type %s and press [Return]: ' % note) @@ -720,7 +744,7 @@ class InfoExtractor(object): 'mature': 17, 'restricted': 19, } - return RATING_TABLE.get(rating.lower(), None) + return RATING_TABLE.get(rating.lower()) def _family_friendly_search(self, html): # See http://schema.org/VideoObject @@ -735,12 +759,48 @@ class InfoExtractor(object): '0': 18, 'false': 18, } - return RATING_TABLE.get(family_friendly.lower(), None) + return RATING_TABLE.get(family_friendly.lower()) def _twitter_search_player(self, html): return self._html_search_meta('twitter:player', html, 'twitter card player') + def _search_json_ld(self, html, video_id, **kwargs): + json_ld = self._search_regex( + r'(?s)]+type=(["\'])application/ld\+json\1[^>]*>(?P.+?)', + html, 'JSON-LD', group='json_ld', **kwargs) + if not json_ld: + return {} + return self._json_ld(json_ld, video_id, fatal=kwargs.get('fatal', True)) + + def _json_ld(self, json_ld, video_id, fatal=True): + if isinstance(json_ld, compat_str): + json_ld = self._parse_json(json_ld, video_id, fatal=fatal) + if not json_ld: + return {} + info = {} + if json_ld.get('@context') == 'http://schema.org': + item_type = json_ld.get('@type') + if item_type == 'TVEpisode': + info.update({ + 'episode': unescapeHTML(json_ld.get('name')), + 'episode_number': int_or_none(json_ld.get('episodeNumber')), + 'description': unescapeHTML(json_ld.get('description')), + }) + part_of_season = json_ld.get('partOfSeason') + if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason': + info['season_number'] = int_or_none(part_of_season.get('seasonNumber')) + part_of_series = json_ld.get('partOfSeries') + if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries': + info['series'] = unescapeHTML(part_of_series.get('name')) + elif item_type == 'Article': + info.update({ + 'timestamp': parse_iso8601(json_ld.get('datePublished')), + 'title': unescapeHTML(json_ld.get('headline')), + 'description': unescapeHTML(json_ld.get('articleBody')), + }) + return dict((k, v) for k, v in info.items() if v is not None) + @staticmethod def _hidden_inputs(html): html = re.sub(r'', '', html) @@ -767,6 +827,12 @@ class InfoExtractor(object): if not formats: raise ExtractorError('No video formats found') + for f in formats: + # Automatically determine tbr when missing based on abr and vbr (improves + # formats sorting in some cases) + if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None: + f['tbr'] = f['abr'] + f['vbr'] + def _formats_key(f): # TODO remove the following workaround from ..utils import determine_ext @@ -956,6 +1022,18 @@ class InfoExtractor(object): return [] m3u8_doc, urlh = res m3u8_url = urlh.geturl() + # A Media Playlist Tag MUST NOT appear in a Master Playlist + # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3 + # The EXT-X-TARGETDURATION tag is REQUIRED for every M3U8 Media Playlists + # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1 + if '#EXT-X-TARGETDURATION' in m3u8_doc: + return [{ + 'url': m3u8_url, + 'format_id': m3u8_id, + 'ext': ext, + 'protocol': entry_protocol, + 'preference': preference, + }] last_info = None last_media = None kv_rex = re.compile( @@ -1000,9 +1078,9 @@ class InfoExtractor(object): # TODO: looks like video codec is not always necessarily goes first va_codecs = codecs.split(',') if va_codecs[0]: - f['vcodec'] = va_codecs[0].partition('.')[0] + f['vcodec'] = va_codecs[0] if len(va_codecs) > 1 and va_codecs[1]: - f['acodec'] = va_codecs[1].partition('.')[0] + f['acodec'] = va_codecs[1] resolution = last_info.get('RESOLUTION') if resolution: width_str, height_str = resolution.split('x') @@ -1106,12 +1184,15 @@ class InfoExtractor(object): formats = [] rtmp_count = 0 http_count = 0 + m3u8_count = 0 + srcs = [] videos = smil.findall(self._xpath_ns('.//video', namespace)) for video in videos: src = video.get('src') - if not src: + if not src or src in srcs: continue + srcs.append(src) bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000) filesize = int_or_none(video.get('size') or video.get('fileSize')) @@ -1143,10 +1224,20 @@ class InfoExtractor(object): continue src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src) + src_url = src_url.strip() if proto == 'm3u8' or src_ext == 'm3u8': - formats.extend(self._extract_m3u8_formats( - src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)) + m3u8_formats = self._extract_m3u8_formats( + src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False) + if len(m3u8_formats) == 1: + m3u8_count += 1 + m3u8_formats[0].update({ + 'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate), + 'tbr': bitrate, + 'width': width, + 'height': height, + }) + formats.extend(m3u8_formats) continue if src_ext == 'f4m': @@ -1179,11 +1270,13 @@ class InfoExtractor(object): return formats def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'): + urls = [] subtitles = {} for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))): src = textstream.get('src') - if not src: + if not src or src in urls: continue + urls.append(src) ext = textstream.get('ext') or determine_ext(src) if not ext: type_ = textstream.get('type') @@ -1244,6 +1337,163 @@ class InfoExtractor(object): }) return entries + def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}): + res = self._download_webpage_handle( + mpd_url, video_id, + note=note or 'Downloading MPD manifest', + errnote=errnote or 'Failed to download MPD manifest', + fatal=fatal) + if res is False: + return [] + mpd, urlh = res + mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group() + + return self._parse_mpd_formats( + compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict) + + def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}): + if mpd_doc.get('type') == 'dynamic': + return [] + + namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None) + + def _add_ns(path): + return self._xpath_ns(path, namespace) + + def is_drm_protected(element): + return element.find(_add_ns('ContentProtection')) is not None + + def extract_multisegment_info(element, ms_parent_info): + ms_info = ms_parent_info.copy() + segment_list = element.find(_add_ns('SegmentList')) + if segment_list is not None: + segment_urls_e = segment_list.findall(_add_ns('SegmentURL')) + if segment_urls_e: + ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e] + initialization = segment_list.find(_add_ns('Initialization')) + if initialization is not None: + ms_info['initialization_url'] = initialization.attrib['sourceURL'] + else: + segment_template = element.find(_add_ns('SegmentTemplate')) + if segment_template is not None: + start_number = segment_template.get('startNumber') + if start_number: + ms_info['start_number'] = int(start_number) + segment_timeline = segment_template.find(_add_ns('SegmentTimeline')) + if segment_timeline is not None: + s_e = segment_timeline.findall(_add_ns('S')) + if s_e: + ms_info['total_number'] = 0 + for s in s_e: + ms_info['total_number'] += 1 + int(s.get('r', '0')) + else: + timescale = segment_template.get('timescale') + if timescale: + ms_info['timescale'] = int(timescale) + segment_duration = segment_template.get('duration') + if segment_duration: + ms_info['segment_duration'] = int(segment_duration) + media_template = segment_template.get('media') + if media_template: + ms_info['media_template'] = media_template + initialization = segment_template.get('initialization') + if initialization: + ms_info['initialization_url'] = initialization + else: + initialization = segment_template.find(_add_ns('Initialization')) + if initialization is not None: + ms_info['initialization_url'] = initialization.attrib['sourceURL'] + return ms_info + + mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration')) + formats = [] + for period in mpd_doc.findall(_add_ns('Period')): + period_duration = parse_duration(period.get('duration')) or mpd_duration + period_ms_info = extract_multisegment_info(period, { + 'start_number': 1, + 'timescale': 1, + }) + for adaptation_set in period.findall(_add_ns('AdaptationSet')): + if is_drm_protected(adaptation_set): + continue + adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info) + for representation in adaptation_set.findall(_add_ns('Representation')): + if is_drm_protected(representation): + continue + representation_attrib = adaptation_set.attrib.copy() + representation_attrib.update(representation.attrib) + mime_type = representation_attrib.get('mimeType') + content_type = mime_type.split('/')[0] if mime_type else representation_attrib.get('contentType') + if content_type == 'text': + # TODO implement WebVTT downloading + pass + elif content_type == 'video' or content_type == 'audio': + base_url = '' + for element in (representation, adaptation_set, period, mpd_doc): + base_url_e = element.find(_add_ns('BaseURL')) + if base_url_e is not None: + base_url = base_url_e.text + base_url + if re.match(r'^https?://', base_url): + break + if mpd_base_url and not re.match(r'^https?://', base_url): + if not mpd_base_url.endswith('/') and not base_url.startswith('/'): + mpd_base_url += '/' + base_url = mpd_base_url + base_url + representation_id = representation_attrib.get('id') + lang = representation_attrib.get('lang') + url_el = representation.find(_add_ns('BaseURL')) + filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None) + f = { + 'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id, + 'url': base_url, + 'width': int_or_none(representation_attrib.get('width')), + 'height': int_or_none(representation_attrib.get('height')), + 'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000), + 'asr': int_or_none(representation_attrib.get('audioSamplingRate')), + 'fps': int_or_none(representation_attrib.get('frameRate')), + 'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'), + 'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'), + 'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None, + 'format_note': 'DASH %s' % content_type, + 'filesize': filesize, + } + representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info) + if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info: + if 'total_number' not in representation_ms_info and 'segment_duration': + segment_duration = float(representation_ms_info['segment_duration']) / float(representation_ms_info['timescale']) + representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration)) + media_template = representation_ms_info['media_template'] + media_template = media_template.replace('$RepresentationID$', representation_id) + media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template) + media_template.replace('$$', '$') + representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number, 'Bandwidth': representation_attrib.get('bandwidth')} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])] + if 'segment_urls' in representation_ms_info: + f.update({ + 'segment_urls': representation_ms_info['segment_urls'], + 'protocol': 'http_dash_segments', + }) + if 'initialization_url' in representation_ms_info: + initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id) + f.update({ + 'initialization_url': initialization_url, + }) + if not f.get('url'): + f['url'] = initialization_url + try: + existing_format = next( + fo for fo in formats + if fo['format_id'] == representation_id) + except StopIteration: + full_info = formats_dict.get(representation_id, {}).copy() + full_info.update(f) + formats.append(full_info) + else: + existing_format.update(f) + else: + self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type) + self._sort_formats(formats) + return formats + def _live_title(self, name): """ Generate the title for a live video """ now = datetime.datetime.now()