ExtractorError,
int_or_none,
PagedList,
- RegexNotFoundError,
unescapeHTML,
unified_strdate,
orderedSet,
write_json_file,
+ uppercase_escape,
)
class YoutubeBaseInfoExtractor(InfoExtractor):
(?:https?://|//)? # http(s):// or protocol-independent URL (optional)
(?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
(?:www\.)?deturl\.com/www\.youtube\.com/|
- (?:www\.)?pwnyoutube\.com|
+ (?:www\.)?pwnyoutube\.com/|
+ (?:www\.)?yourepeat\.com/|
tube\.majestyc\.net/|
youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
(?:.*?\#/)? # handle anchor (#/) redirect urls
(?: # the various things that can precede the ID:
(?:(?:v|embed|e)/) # v/ or embed/ or e/
|(?: # or the v= param in all its forms
- (?:(?:watch|movie)(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
+ (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
(?:\?|\#!?) # the params delimiter ? or # or #!
(?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
v=
u"format": "141",
},
},
+ # DASH manifest with encrypted signature
+ {
+ u'url': u'https://www.youtube.com/watch?v=IB3lcPjvWLA',
+ u'info_dict': {
+ u'id': u'IB3lcPjvWLA',
+ u'ext': u'm4a',
+ u'title': u'Afrojack - The Spark ft. Spree Wilson',
+ u'description': u'md5:3199ed45ee8836572865580804d7ac0f',
+ u'uploader': u'AfrojackVEVO',
+ u'uploader_id': u'AfrojackVEVO',
+ u'upload_date': u'20131011',
+ },
+ u"params": {
+ u'youtube_include_dash_manifest': True,
+ u'format': '141',
+ },
+ },
]
return a % b
m = re.match(
- r'^(?P<func>[a-zA-Z]+)\((?P<args>[a-z0-9,]+)\)$', expr)
+ r'^(?P<func>[a-zA-Z
$]+)\((?P<args>[a-z0-9,]+)\)$', expr)
if m:
fname = m.group('func')
if fname not in functions:
self._downloader.report_warning(err_msg)
return {}
- def _extract_id(self, url):
- mobj = re.match(self._VALID_URL, url, re.VERBOSE)
+ @classmethod
+ def extract_id(cls, url):
+ mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url)
video_id = mobj.group(2)
mobj = re.search(self._NEXT_URL_RE, url)
if mobj:
url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
- video_id = self._extract_id(url)
+ video_id = self.extract_id(url)
# Get video webpage
url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
if not mobj:
raise ValueError('Could not find vevo ID')
- info = json.loads(mobj.group(1))
- args = info['args']
+ ytplayer_config = json.loads(mobj.group(1))
+ args = ytplayer_config['args']
# Easy way to know if the 's' value is in url_encoded_fmt_stream_map
# this signatures are encrypted
if 'url_encoded_fmt_stream_map' not in args:
raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
# Look for the DASH manifest
- dash_manifest_url_lst = video_info.get('dashmpd')
- if (dash_manifest_url_lst and dash_manifest_url_lst[0] and
- self._downloader.params.get('youtube_include_dash_manifest', False)):
+ if (self._downloader.params.get('youtube_include_dash_manifest', False)):
try:
+ # The DASH manifest used needs to be the one from the original video_webpage.
+ # The one found in get_video_info seems to be using different signatures.
+ # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage.
+ # Luckily, it seems, this case uses some kind of default signature (len == 86), so the
+ # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here.
+ if age_gate:
+ dash_manifest_url = video_info.get('dashmpd')[0]
+ else:
+ dash_manifest_url = ytplayer_config['args']['dashmpd']
+ def decrypt_sig(mobj):
+ s = mobj.group(1)
+ dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
+ return '/signature/%s' % dec_s
+ dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
dash_doc = self._download_xml(
- dash_manifest_url_lst[0], video_id,
+ dash_manifest_url, video_id,
note=u'Downloading DASH manifest',
errnote=u'Could not download DASH manifest')
for r in dash_doc.findall(u'.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
IE_DESC = u'YouTube.com playlists'
- _VALID_URL = r"""(?:
+ _VALID_URL = r"""(?x)(?:
(?:https?://)?
(?:\w+\.)?
youtube\.com/
\? (?:.*?&)*? (?:p|a|list)=
| p/
)
- ((?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,})
+ (
+ (?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}
+ # Top tracks, they can also include dots
+ |(?:MC)[\w\.]*
+ )
.*
|
((?:PL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
)"""
- _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s'
+ _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
_MORE_PAGES_INDICATOR = r'data-link-type="next"'
- _VIDEO_RE = r'href="/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)'
+ _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)'
IE_NAME = u'youtube:playlist'
- @classmethod
- def suitable(cls, url):
- """Receives a URL and returns True if suitable for this IE."""
- return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
-
def _real_initialize(self):
self._login()
# the id of the playlist is just 'RD' + video_id
url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix')
- title_span = (get_element_by_attribute('class', 'title long-title', webpage) or
- get_element_by_attribute('class', 'title ', webpage))
+ search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
+ title_span = (search_title('playlist-title') or
+ search_title('title long-title') or search_title('title'))
title = clean_html(title_span)
- video_re = r'data-index="\d+".*?href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s' % re.escape(playlist_id)
- ids = orderedSet(re.findall(video_re, webpage))
+ video_re = r'''(?x)data-video-username="(.*?)".*?
+ href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re.escape(playlist_id)
+ matches = orderedSet(re.findall(video_re, webpage, flags=re.DOTALL))
+ # Some of the videos may have been deleted, their username field is empty
+ ids = [video_id for (username, video_id) in matches if username]
url_results = self._ids_to_results(ids)
return self.playlist_result(url_results, playlist_id, title)
def _real_extract(self, url):
# Extract playlist id
- mobj = re.match(self._VALID_URL, url, re.VERBOSE)
+ mobj = re.match(self._VALID_URL, url)
if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url)
playlist_id = mobj.group(1) or mobj.group(2)
raise ExtractorError(u'For downloading YouTube.com top lists, use '
u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
+ url = self._TEMPLATE_URL % playlist_id
+ page = self._download_webpage(url, playlist_id)
+ more_widget_html = content_html = page
+
# Extract the video ids from the playlist pages
ids = []
for page_num in itertools.count(1):
- url = self._TEMPLATE_URL % (playlist_id, page_num)
- page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
- matches = re.finditer(self._VIDEO_RE, page)
+ matches = re.finditer(self._VIDEO_RE, content_html)
# We remove the duplicates and the link with index 0
# (it's not the first video of the playlist)
new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
ids.extend(new_ids)
- if re.search(self._MORE_PAGES_INDICATOR, page) is None:
+ mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+ if not mobj:
break
- try:
- playlist_title = self._og_search_title(page)
- except RegexNotFoundError:
- self.report_warning(
- u'Playlist page is missing OpenGraph title, falling back ...',
- playlist_id)
- playlist_title = self._html_search_regex(
- r'<h1 class="pl-header-title">(.*?)</h1>', page, u'title')
+ more = self._download_json(
+ 'https://youtube.com/%s' % mobj.group('more'), playlist_id, 'Downloading page #%s' % page_num)
+ content_html = more['content_html']
+ more_widget_html = more['load_more_widget_html']
+
+ playlist_title = self._html_search_regex(
+ r'<h1 class="pl-header-title">\s*(.*?)\s*</h1>', page, u'title')
url_results = self._ids_to_results(ids)
return self.playlist_result(url_results, playlist_id, playlist_title)
# Download all channel pages using the json-based channel_ajax query
for pagenum in itertools.count(1):
url = self._MORE_PAGES_URL % (pagenum, channel_id)
- page = self._download_webpage(url, channel_id,
- u'Downloading page #%s' % pagenum)
-
- page = json.loads(page)
-
+ page = self._download_json(
+ url, channel_id, note=u'Downloading page #%s' % pagenum,
+ transform_source=uppercase_escape)
+
ids_in_page = self.extract_videos_from_page(page['content_html'])
video_ids.extend(ids_in_page)
api_response = data['data']
if 'items' not in api_response:
- raise ExtractorError(u'[youtube] No video results')
+ raise ExtractorError(
+ u'[youtube] No video results', expected=True)
new_ids = list(video['id'] for video in api_response['items'])
video_ids += new_ids
IE_NAME = 'youtube:truncated_url'
IE_DESC = False # Do not list
_VALID_URL = r'''(?x)
- (?:https?://)?[^/]+/watch\?feature=[a-z_]+$|
+ (?:https?://)?[^/]+/watch\?(?:feature=[a-z_]+)?$|
(?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$
'''
projects / youtube-dl / blobdiff