#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
import datetime
import netrc
import os
import random
import math
-from utils import *
+from .utils import *
class InfoExtractor(object):
# Extension
video_extension = self._video_extensions.get(format_param, 'flv')
- video_format = '{} - {}'.format(format_param if format_param else video_extension,
- self._video_dimensions.get(format_param, '???'))
+ video_format = '{0} - {1}'.format(format_param if format_param else video_extension,
+ self._video_dimensions.get(format_param, '???'))
results.append({
'id': video_id,
self.report_extraction(video_id)
# Extract the config JSON
- config = webpage.split(' = {config:')[1].split(',assets:')[0]
try:
+ config = webpage.split(' = {config:')[1].split(',assets:')[0]
config = json.loads(config)
except:
self._downloader.trouble(u'ERROR: unable to extract info section')
class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists."""
- _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+ _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
- _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
+ _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
IE_NAME = u'youtube:playlist'
def __init__(self, downloader=None):
url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
request = compat_urllib_request.Request(url)
try:
- page = compat_urllib_request.urlopen(request).read()
+ page = compat_urllib_request.urlopen(request).read().decode('utf8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return
ids_in_page.append(mobj.group(1))
video_ids.extend(ids_in_page)
- if re.search(self._MORE_PAGES_INDICATOR, page) is None:
+ if self._MORE_PAGES_INDICATOR not in page:
break
pagenum = pagenum + 1
+ total = len(video_ids)
+
playliststart = self._downloader.params.get('playliststart', 1) - 1
playlistend = self._downloader.params.get('playlistend', -1)
if playlistend == -1:
else:
video_ids = video_ids[playliststart:playlistend]
+ if len(video_ids) == total:
+ self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total))
+ else:
+ self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids)))
+
for id in video_ids:
self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
return
_VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
_TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
- _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
+ _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
IE_NAME = u'youtube:channel'
def report_download_page(self, channel_id, pagenum):
url = self._TEMPLATE_URL % (channel_id, pagenum)
request = compat_urllib_request.Request(url)
try:
- page = compat_urllib_request.urlopen(request).read()
+ page = compat_urllib_request.urlopen(request).read().decode('utf8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return
ids_in_page.append(mobj.group(1))
video_ids.extend(ids_in_page)
- if re.search(self._MORE_PAGES_INDICATOR, page) is None:
+ if self._MORE_PAGES_INDICATOR not in page:
break
pagenum = pagenum + 1
+ self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
+
for id in video_ids:
self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
return
else:
cchar = '?'
json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
- request = compat_urllib_request.Request(json_url.encode('utf-8'))
+ request = compat_urllib_request.Request(json_url)
self.report_extraction(mobj.group(1))
info = None
try:
return
if info is None: # Regular URL
try:
- json_code = urlh.read()
+ json_code_bytes = urlh.read()
+ json_code = json_code_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
return
class ComedyCentralIE(InfoExtractor):
"""Information extractor for The Daily Show and Colbert Report """
- _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
+ # urls can be abbreviations like :thedailyshow or :colbert
+ # urls for episodes like:
+ # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
+ # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
+ # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
+ _VALID_URL = r"""^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport)
+ |(https?://)?(www\.)?
+ (?P<showname>thedailyshow|colbertnation)\.com/
+ (full-episodes/(?P<episode>.*)|
+ (?P<clip>
+ (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
+ |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)))))
+ $"""
IE_NAME = u'comedycentral'
_available_formats = ['3500', '2200', '1700', '1200', '750', '400']
'400': '384x216',
}
+ def suitable(self, url):
+ """Receives a URL and returns True if suitable for this IE."""
+ return re.match(self._VALID_URL, url, re.VERBOSE) is not None
+
def report_extraction(self, episode_id):
self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
+ mobj = re.match(self._VALID_URL, url, re.VERBOSE)
if mobj is None:
self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
return
url = u'http://www.thedailyshow.com/full-episodes/'
else:
url = u'http://www.colbertnation.com/full-episodes/'
- mobj = re.match(self._VALID_URL, url)
+ mobj = re.match(self._VALID_URL, url, re.VERBOSE)
assert mobj is not None
- dlNewest = not mobj.group('episode')
- if dlNewest:
- epTitle = mobj.group('showname')
+ if mobj.group('clip'):
+ if mobj.group('showname') == 'thedailyshow':
+ epTitle = mobj.group('tdstitle')
+ else:
+ epTitle = mobj.group('cntitle')
+ dlNewest = False
else:
- epTitle = mobj.group('episode')
+ dlNewest = not mobj.group('episode')
+ if dlNewest:
+ epTitle = mobj.group('showname')
+ else:
+ epTitle = mobj.group('episode')
req = compat_urllib_request.Request(url)
self.report_extraction(epTitle)
return
if dlNewest:
url = htmlHandle.geturl()
- mobj = re.match(self._VALID_URL, url)
+ mobj = re.match(self._VALID_URL, url, re.VERBOSE)
if mobj is None:
self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
return
return
epTitle = mobj.group('episode')
- mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
+ mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', html)
if len(mMovieParams) == 0:
# The Colbert Report embeds the information in a without
# a URL prefix; so extract the alternate reference
# and then add the URL prefix manually.
- altMovieParams = re.findall('data-mgid="([^"]*episode.*?:.*?)"', html)
+ altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video).*?:.*?)"', html)
if len(altMovieParams) == 0:
self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
return
class CollegeHumorIE(InfoExtractor):
"""Information extractor for collegehumor.com"""
+ _WORKING = False
_VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
IE_NAME = u'collegehumor'
- def report_webpage(self, video_id):
+ def report_manifest(self, video_id):
"""Report information extraction."""
- self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+ self._downloader.to_screen(u'[%s] %s: Downloading XML manifest' % (self.IE_NAME, video_id))
def report_extraction(self, video_id):
"""Report information extraction."""
return
video_id = mobj.group('videoid')
- self.report_webpage(video_id)
- request = compat_urllib_request.Request(url)
- try:
- webpage = compat_urllib_request.urlopen(request).read()
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
- return
-
- m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
- if m is None:
- self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
- return
- internal_video_id = m.group('internalvideoid')
-
info = {
'id': video_id,
- 'internal_id': internal_video_id,
'uploader': None,
'upload_date': None,
}
self.report_extraction(video_id)
- xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
+ xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id
try:
metaXml = compat_urllib_request.urlopen(xmlUrl).read()
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
videoNode = mdoc.findall('./video')[0]
info['description'] = videoNode.findall('./description')[0].text
info['title'] = videoNode.findall('./caption')[0].text
- info['url'] = videoNode.findall('./file')[0].text
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
- info['ext'] = info['url'].rpartition('.')[2]
+ manifest_url = videoNode.findall('./file')[0].text
except IndexError:
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
return
+ manifest_url += '?hdcore=2.10.3'
+ self.report_manifest(video_id)
+ try:
+ manifestXml = compat_urllib_request.urlopen(manifest_url).read()
+ except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
+ return
+
+ adoc = xml.etree.ElementTree.fromstring(manifestXml)
+ try:
+ media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
+ node_id = media_node.attrib['url']
+ video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
+ except IndexError as err:
+ self._downloader.trouble(u'\nERROR: Invalid manifest file')
+ return
+
+ url_pr = compat_urllib_parse_urlparse(manifest_url)
+ url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1'
+
+ info['url'] = url
+ info['ext'] = 'f4f'
return [info]
def __init__(self, downloader=None):
InfoExtractor.__init__(self, downloader)
- def report_webpage(self, video_id):
+ def report_resolve(self, video_id):
"""Report information extraction."""
- self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+ self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
def report_extraction(self, video_id):
"""Report information extraction."""
- self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+ self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
return
# extract uploader (which is in the url)
- uploader = mobj.group(1).decode('utf-8')
+ uploader = mobj.group(1)
# extract simple title (uploader + slug of song title)
- slug_title = mobj.group(2).decode('utf-8')
+ slug_title = mobj.group(2)
simple_title = uploader + u'-' + slug_title
- self.report_webpage('%s/%s' % (uploader, slug_title))
+ self.report_resolve('%s/%s' % (uploader, slug_title))
- request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+ url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
+ resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
+ request = compat_urllib_request.Request(resolv_url)
try:
- webpage = compat_urllib_request.urlopen(request).read()
+ info_json_bytes = compat_urllib_request.urlopen(request).read()
+ info_json = info_json_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return
+ info = json.loads(info_json)
+ video_id = info['id']
self.report_extraction('%s/%s' % (uploader, slug_title))
- # extract uid and stream token that soundcloud hands out for access
- mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
- if mobj:
- video_id = mobj.group(1)
- stream_token = mobj.group(2)
-
- # extract unsimplified title
- mobj = re.search('"title":"(.*?)",', webpage)
- if mobj:
- title = mobj.group(1).decode('utf-8')
- else:
- title = simple_title
-
- # construct media url (with uid/token)
- mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
- mediaURL = mediaURL % (video_id, stream_token)
-
- # description
- description = u'No description available'
- mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
- if mobj:
- description = mobj.group(1)
-
- # upload date
- upload_date = None
- mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
- if mobj:
- try:
- upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
- except Exception as err:
- self._downloader.to_stderr(compat_str(err))
+ streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
+ request = compat_urllib_request.Request(streams_url)
+ try:
+ stream_json_bytes = compat_urllib_request.urlopen(request).read()
+ stream_json = stream_json_bytes.decode('utf-8')
+ except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ return
- # for soundcloud, a request to a cross domain is required for cookies
- request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+ streams = json.loads(stream_json)
+ mediaURL = streams['http_mp3_128_url']
return [{
- 'id': video_id.decode('utf-8'),
+ 'id': info['id'],
'url': mediaURL,
- 'uploader': uploader.decode('utf-8'),
- 'upload_date': upload_date,
- 'title': title,
+ 'uploader': info['user']['username'],
+ 'upload_date': info['created_at'],
+ 'title': info['title'],
'ext': u'mp3',
- 'description': description.decode('utf-8')
+ 'description': info['description'],
}]
projects / youtube-dl / blobdiff