*.m4a
*.m4v
*.part
+*.swp
test/testdata
.tox
cleanall: clean
rm -f youtube-dl youtube-dl.exe
-PREFIX=/usr/local
-BINDIR=$(PREFIX)/bin
-MANDIR=$(PREFIX)/man
-PYTHON=/usr/bin/env python
+PREFIX ?= /usr/local
+BINDIR ?= $(PREFIX)/bin
+MANDIR ?= $(PREFIX)/man
+PYTHON ?= /usr/bin/env python
# set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
ifeq ($(PREFIX),/usr)
If you do not have curl, you can alternatively use a recent wget:
- sudo wget https://yt-dl.org/downloads/2014.05.13/youtube-dl -O /usr/local/bin/youtube-dl
+ sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
sudo chmod a+x /usr/local/bin/youtube-dl
Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
+OS X users can install **youtube-dl** with [Homebrew](http://brew.sh/).
+
+ brew install youtube-dl
+
+You can also use pip:
+
+ sudo pip install youtube-dl
+
Alternatively, refer to the developer instructions below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
# DESCRIPTION
playlist or the command line) if an error
occurs
--dump-user-agent display the current browser identification
- --user-agent UA specify a custom user agent
- --referer REF specify a custom referer, use if the video
- access is restricted to one domain
- --add-header FIELD:VALUE specify a custom HTTP header and its value,
- separated by a colon ':'. You can use this
- option multiple times
--list-extractors List all supported extractors and the URLs
they would handle
--extractor-descriptions Output descriptions of all supported
--proxy URL Use the specified HTTP/HTTPS proxy. Pass in
an empty string (--proxy "") for direct
connection
- --no-check-certificate Suppress HTTPS certificate validation.
- --prefer-insecure Use an unencrypted connection to retrieve
- information about the video. (Currently
- supported only for YouTube)
- --cache-dir DIR Location in the filesystem where youtube-dl
- can store some downloaded information
- permanently. By default $XDG_CACHE_HOME
- /youtube-dl or ~/.cache/youtube-dl . At the
- moment, only YouTube player files (for
- videos with obfuscated signatures) are
- cached, but that may change.
- --no-cache-dir Disable filesystem caching
--socket-timeout None Time to wait before giving up, in seconds
- --bidi-workaround Work around terminals that lack
- bidirectional text support. Requires bidiv
- or fribidi executable in PATH
--default-search PREFIX Use this prefix for unqualified URLs. For
example "gvsearch2:" downloads two videos
from google videos for youtube-dl "large
apple". Use the value "auto" to let
- youtube-dl guess. The default value "error"
- just throws an error.
+ youtube-dl guess ("auto_warning" to emit a
+ warning when guessing). "error" just throws
+ an error. The default value "fixup_error"
+ repairs broken URLs, but emits an error if
+ this is not possible instead of searching.
--ignore-config Do not read configuration files. When given
in the global configuration file /etc
/youtube-dl.conf: do not read the user
configuration in ~/.config/youtube-dl.conf
(%APPDATA%/youtube-dl/config.txt on
Windows)
- --encoding ENCODING Force the specified encoding (experimental)
## Video Selection:
--playlist-start NUMBER playlist video to start at (default is 1)
of SIZE.
## Filesystem Options:
- -t, --title use title in file name (default)
+ -a, --batch-file FILE file containing URLs to download ('-' for
+ stdin)
--id use only video ID in file name
- -l, --literal [deprecated] alias of --title
-A, --auto-number number downloaded files starting from 00000
-o, --output TEMPLATE output filename template. Use %(title)s to
get the title, %(uploader)s for the
--restrict-filenames Restrict filenames to only ASCII
characters, and avoid "&" and spaces in
filenames
- -a, --batch-file FILE file containing URLs to download ('-' for
- stdin)
- --load-info FILE json file containing the video information
- (created with the "--write-json" option)
+ -t, --title [deprecated] use title in file name
+ (default)
+ -l, --literal [deprecated] alias of --title
-w, --no-overwrites do not overwrite files
-c, --continue force resume of partially downloaded files.
By default, youtube-dl will resume
downloads if possible.
--no-continue do not resume partially downloaded files
(restart from beginning)
- --cookies FILE file to read cookies from and dump cookie
- jar in
--no-part do not use .part files
--no-mtime do not use the Last-modified header to set
the file modification time
--write-annotations write video annotations to a .annotation
file
--write-thumbnail write thumbnail image to disk
+ --load-info FILE json file containing the video information
+ (created with the "--write-json" option)
+ --cookies FILE file to read cookies from and dump cookie
+ jar in
+ --cache-dir DIR Location in the filesystem where youtube-dl
+ can store some downloaded information
+ permanently. By default $XDG_CACHE_HOME
+ /youtube-dl or ~/.cache/youtube-dl . At the
+ moment, only YouTube player files (for
+ videos with obfuscated signatures) are
+ cached, but that may change.
+ --no-cache-dir Disable filesystem caching
+ --rm-cache-dir Delete all filesystem cache files
## Verbosity / Simulation Options:
-q, --quiet activates quiet mode
problems
--print-traffic Display sent and read HTTP traffic
+## Workarounds:
+ --encoding ENCODING Force the specified encoding (experimental)
+ --no-check-certificate Suppress HTTPS certificate validation.
+ --prefer-insecure Use an unencrypted connection to retrieve
+ information about the video. (Currently
+ supported only for YouTube)
+ --user-agent UA specify a custom user agent
+ --referer REF specify a custom referer, use if the video
+ access is restricted to one domain
+ --add-header FIELD:VALUE specify a custom HTTP header and its value,
+ separated by a colon ':'. You can use this
+ option multiple times
+ --bidi-workaround Work around terminals that lack
+ bidirectional text support. Requires bidiv
+ or fribidi executable in PATH
+
## Video Format Options:
-f, --format FORMAT video format code, specify the order of
preference using slashes: "-f 22/17/18".
## Authentication Options:
-u, --username USERNAME account username
-p, --password PASSWORD account password
+ -2, --twofactor TWOFACTOR two-factor auth code
-n, --netrc use .netrc authentication data
--video-password PASSWORD video password (vimeo, smotri)
postprocessors (default)
--prefer-ffmpeg Prefer ffmpeg over avconv for running the
postprocessors
+ --exec CMD Execute a command on the file after
+ downloading, similar to find's -exec
+ syntax. Example: --exec 'adb push {}
+ /sdcard/Music/ && rm {}'
# CONFIGURATION
In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
- $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
- youtube-dl test video ''_ä↭𝕐.mp4 # All kinds of weird characters
- $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
- youtube-dl_test_video_.mp4 # A simple file name
+```bash
+$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
+youtube-dl test video ''_ä↭𝕐.mp4 # All kinds of weird characters
+$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
+youtube-dl_test_video_.mp4 # A simple file name
+```
# VIDEO SELECTION
Examples:
- # Download only the videos uploaded in the last 6 months
- $ youtube-dl --dateafter now-6months
+```bash
+# Download only the videos uploaded in the last 6 months
+$ youtube-dl --dateafter now-6months
- # Download only the videos uploaded on January 1, 1970
- $ youtube-dl --date 19700101
+# Download only the videos uploaded on January 1, 1970
+$ youtube-dl --date 19700101
- $ # will only download the videos uploaded in the 200x decade
- $ youtube-dl --dateafter 20000101 --datebefore 20091231
+$ # will only download the videos uploaded in the 200x decade
+$ youtube-dl --dateafter 20000101 --datebefore 20091231
+```
# FAQ
2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor`
4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
-
- # coding: utf-8
- from __future__ import unicode_literals
-
- import re
-
- from .common import InfoExtractor
-
-
- class YourExtractorIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)'
- _TEST = {
- 'url': 'http://yourextractor.com/watch/42',
- 'md5': 'TODO: md5 sum of the first 10KiB of the video file',
- 'info_dict': {
- 'id': '42',
- 'ext': 'mp4',
- 'title': 'Video title goes here',
- # TODO more properties, either as:
- # * A value
- # * MD5 checksum; start the string with md5:
- # * A regular expression; start the string with re:
- # * Any Python type (for example int or float)
- }
+ ```python
+ # coding: utf-8
+ from __future__ import unicode_literals
+
+ import re
+
+ from .common import InfoExtractor
+
+
+ class YourExtractorIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'http://yourextractor.com/watch/42',
+ 'md5': 'TODO: md5 sum of the first 10KiB of the video file',
+ 'info_dict': {
+ 'id': '42',
+ 'ext': 'mp4',
+ 'title': 'Video title goes here',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ # TODO more properties, either as:
+ # * A value
+ # * MD5 checksum; start the string with md5:
+ # * A regular expression; start the string with re:
+ # * Any Python type (for example int or float)
}
+ }
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
-
- # TODO more code goes here, for example ...
- webpage = self._download_webpage(url, video_id)
- title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
-
- return {
- 'id': video_id,
- 'title': title,
- # TODO more properties (see youtube_dl/extractor/common.py)
- }
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ # TODO more code goes here, for example ...
+ webpage = self._download_webpage(url, video_id)
+ title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+ return {
+ 'id': video_id,
+ 'title': title,
+ # TODO more properties (see youtube_dl/extractor/common.py)
+ }
+ ```
5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
9. When the tests pass, [add](https://www.kernel.org/pub/software/scm/git/docs/git-add.html) the new files and [commit](https://www.kernel.org/pub/software/scm/git/docs/git-commit.html) them and [push](https://www.kernel.org/pub/software/scm/git/docs/git-push.html) the result, like this:
match_rex = re.compile(match_str)
self.assertTrue(
- isinstance(got, compat_str) and match_rex.match(got),
+ isinstance(got, compat_str),
+ u'Expected a %s object, but got %s for field %s' % (
+ compat_str.__name__, type(got).__name__, info_field))
+ self.assertTrue(
+ match_rex.match(got),
u'field %s (value: %r) should match %r' % (info_field, got, match_str))
elif isinstance(expected, type):
got = got_dict.get(info_field)
u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
# Check for the presence of mandatory fields
- for key in ('id', 'url', 'title', 'ext'):
- self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
+ if got_dict.get('_type') != 'playlist':
+ for key in ('id', 'url', 'title', 'ext'):
+ self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
# Check for mandatory fields that are automatically set by YoutubeDL
for key in ['webpage_url', 'extractor', 'extractor_key']:
self.assertTrue(got_dict.get(key), u'Missing field: %s' % key)
def assertRegexpMatches(self, text, regexp, msg=None):
- if hasattr(self, 'assertRegexpMatches'):
- return self.assertRegexpMatches(text, regexp, msg)
+ if hasattr(self, 'assertRegexp'):
+ return self.assertRegexp(text, regexp, msg)
else:
m = re.match(regexp, text)
if not m:
else:
msg = note + ', ' + msg
self.assertTrue(m, msg)
+
+
+def assertGreaterEqual(self, got, expected, msg=None):
+ if not (got >= expected):
+ if msg is None:
+ msg = '%r not greater than or equal to %r' % (got, expected)
+ self.assertTrue(got >= expected, msg)
"rejecttitle": null,
"retries": 10,
"simulate": false,
- "skip_download": false,
"subtitleslang": null,
"subtitlesformat": "srt",
"test": true,
--- /dev/null
+// input: [["a", "b", "c", "d"]]
+// output: ["c", "b", "a", "d"]
+
+package {
+public class ArrayAccess {
+ public static function main(ar:Array):Array {
+ var aa:ArrayAccess = new ArrayAccess();
+ return aa.f(ar, 2);
+ }
+
+ private function f(ar:Array, num:Number):Array{
+ var x:String = ar[0];
+ var y:String = ar[num % ar.length];
+ ar[0] = y;
+ ar[num] = x;
+ return ar;
+ }
+}
+}
--- /dev/null
+// input: []
+// output: 121
+
+package {
+public class ClassCall {
+ public static function main():int{
+ var f:OtherClass = new OtherClass();
+ return f.func(100,20);
+ }
+}
+}
+
+class OtherClass {
+ public function func(x: int, y: int):int {
+ return x+y+1;
+ }
+}
--- /dev/null
+// input: []
+// output: 0
+
+package {
+public class ClassConstruction {
+ public static function main():int{
+ var f:Foo = new Foo();
+ return 0;
+ }
+}
+}
+
+class Foo {
+
+}
--- /dev/null
+// input: [1, 2]
+// output: 3
+
+package {
+public class LocalVars {
+ public static function main(a:int, b:int):int{
+ var c:int = a + b + b;
+ var d:int = c - b;
+ var e:int = d;
+ return e;
+ }
+}
+}
--- /dev/null
+// input: []
+// output: 9
+
+package {
+public class PrivateCall {
+ public static function main():int{
+ var f:OtherClass = new OtherClass();
+ return f.func();
+ }
+}
+}
+
+class OtherClass {
+ private function pf():int {
+ return 9;
+ }
+
+ public function func():int {
+ return this.pf();
+ }
+}
--- /dev/null
+// input: [1]
+// output: 1
+
+package {
+public class StaticAssignment {
+ public static var v:int;
+
+ public static function main(a:int):int{
+ v = a;
+ return v;
+ }
+}
+}
--- /dev/null
+// input: []
+// output: 1
+
+package {
+public class StaticRetrieval {
+ public static var v:int;
+
+ public static function main():int{
+ if (v) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+}
+}
'138', '137', '248', '136', '247', '135', '246',
'245', '244', '134', '243', '133', '242', '160',
# Dash audio
- '141', '172', '140', '139', '171',
+ '141', '172', '140', '171', '139',
]
for f1id, f2id in zip(order, order[1:]):
FacebookIE,
gen_extractors,
JustinTVIE,
- PBSIE,
YoutubeIE,
)
def test_facebook_matching(self):
self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268'))
+ self.assertTrue(FacebookIE.suitable('https://www.facebook.com/cindyweather?fref=ts#!/photo.php?v=10152183998945793'))
def test_no_duplicates(self):
ies = gen_extractors()
if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'):
self.assertTrue(ie.suitable(url), '%s should match URL %r' % (type(ie).__name__, url))
else:
- self.assertFalse(ie.suitable(url), '%s should not match URL %r' % (type(ie).__name__, url))
+ self.assertFalse(
+ ie.suitable(url),
+ '%s should not match URL %r . That URL belongs to %s.' % (type(ie).__name__, url, tc['name']))
def test_keywords(self):
self.assertMatch(':ytsubs', ['youtube:subscriptions'])
self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['PBS'])
self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['PBS'])
- def test_ComedyCentralShows(self):
- self.assertMatch(
- 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
- ['ComedyCentralShows'])
- self.assertMatch(
- 'http://thecolbertreport.cc.com/videos/29w6fx/-realhumanpraise-for-fox-news',
- ['ComedyCentralShows'])
- self.assertMatch(
- 'http://thecolbertreport.cc.com/videos/gh6urb/neil-degrasse-tyson-pt--1?xrs=eml_col_031114',
- ['ComedyCentralShows'])
- self.assertMatch(
- 'http://thedailyshow.cc.com/guests/michael-lewis/3efna8/exclusive---michael-lewis-extended-interview-pt--3',
- ['ComedyCentralShows'])
- self.assertMatch(
- 'http://thedailyshow.cc.com/episodes/sy7yv0/april-8--2014---denis-leary',
- ['ComedyCentralShows'])
- self.assertMatch(
- 'http://thecolbertreport.cc.com/episodes/8ase07/april-8--2014---jane-goodall',
- ['ComedyCentralShows'])
- self.assertMatch(
- 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights',
- ['ComedyCentralShows'])
- self.assertMatch(
- 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food',
- ['ComedyCentralShows'])
-
def test_yahoo_https(self):
# https://github.com/rg3/youtube-dl/issues/2701
self.assertMatch(
--- /dev/null
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import shutil
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from test.helper import FakeYDL
+from youtube_dl.cache import Cache
+
+
+def _is_empty(d):
+ return not bool(os.listdir(d))
+
+
+def _mkdir(d):
+ if not os.path.exists(d):
+ os.mkdir(d)
+
+
+class TestCache(unittest.TestCase):
+ def setUp(self):
+ TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+ TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
+ _mkdir(TESTDATA_DIR)
+ self.test_dir = os.path.join(TESTDATA_DIR, 'cache_test')
+ self.tearDown()
+
+ def tearDown(self):
+ if os.path.exists(self.test_dir):
+ shutil.rmtree(self.test_dir)
+
+ def test_cache(self):
+ ydl = FakeYDL({
+ 'cachedir': self.test_dir,
+ })
+ c = Cache(ydl)
+ obj = {'x': 1, 'y': ['ä', '\\a', True]}
+ self.assertEqual(c.load('test_cache', 'k.'), None)
+ c.store('test_cache', 'k.', obj)
+ self.assertEqual(c.load('test_cache', 'k2'), None)
+ self.assertFalse(_is_empty(self.test_dir))
+ self.assertEqual(c.load('test_cache', 'k.'), obj)
+ self.assertEqual(c.load('test_cache', 'y'), None)
+ self.assertEqual(c.load('test_cache2', 'k.'), None)
+ c.remove()
+ self.assertFalse(os.path.exists(self.test_dir))
+ self.assertEqual(c.load('test_cache', 'k.'), None)
+
+
+if __name__ == '__main__':
+ unittest.main()
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from test.helper import (
+ assertGreaterEqual,
get_params,
gettestcases,
expect_info_dict,
- md5,
try_rm,
report_warning,
)
import youtube_dl.YoutubeDL
from youtube_dl.utils import (
compat_http_client,
- compat_str,
compat_urllib_error,
compat_HTTPError,
DownloadError,
ExtractorError,
+ format_bytes,
UnavailableVideoError,
)
from youtube_dl.extractor import get_info_extractor
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
+ is_playlist = any(k.startswith('playlist') for k in test_case)
+ test_cases = test_case.get(
+ 'playlist', [] if is_playlist else [test_case])
+
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
- if 'playlist' not in test_case:
- info_dict = test_case.get('info_dict', {})
- if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
+
+ for tc in test_cases:
+ info_dict = tc.get('info_dict', {})
+ if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
+
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
return
params = get_params(test_case.get('params', {}))
+ if is_playlist and 'playlist' not in test_case:
+ params.setdefault('extract_flat', True)
+ params.setdefault('skip_download', True)
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
- test_cases = test_case.get('playlist', [test_case])
- def try_rm_tcs_files():
- for tc in test_cases:
+ res_dict = None
+ def try_rm_tcs_files(tcs=None):
+ if tcs is None:
+ tcs = test_cases
+ for tc in tcs:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_num = 1
while True:
try:
- ydl.download([test_case['url']])
+ # We're not using .download here sine that is just a shim
+ # for outside error handling, and returns the exit code
+ # instead of the result dict.
+ res_dict = ydl.extract_info(test_case['url'])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
else:
break
+ if is_playlist:
+ self.assertEqual(res_dict['_type'], 'playlist')
+ expect_info_dict(self, test_case.get('info_dict', {}), res_dict)
+ if 'playlist_mincount' in test_case:
+ assertGreaterEqual(
+ self,
+ len(res_dict['entries']),
+ test_case['playlist_mincount'],
+ 'Expected at least %d in playlist %s, but got only %d' % (
+ test_case['playlist_mincount'], test_case['url'],
+ len(res_dict['entries'])))
+ if 'playlist_count' in test_case:
+ self.assertEqual(
+ len(res_dict['entries']),
+ test_case['playlist_count'],
+ 'Expected %d entries in playlist %s, but got %d.' % (
+ test_case['playlist_count'],
+ test_case['url'],
+ len(res_dict['entries']),
+ ))
+ if 'playlist_duration_sum' in test_case:
+ got_duration = sum(e['duration'] for e in res_dict['entries'])
+ self.assertEqual(
+ test_case['playlist_duration_sum'], got_duration)
+
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
+ expected_minsize = tc.get('file_minsize', 10000)
+ if expected_minsize is not None:
+ if params.get('test'):
+ expected_minsize = max(expected_minsize, 10000)
+ got_fsize = os.path.getsize(tc_filename)
+ assertGreaterEqual(
+ self, got_fsize, expected_minsize,
+ 'Expected %s to be at least %s, but it\'s only %s ' %
+ (tc_filename, format_bytes(expected_minsize),
+ format_bytes(got_fsize)))
+ if 'md5' in tc:
+ md5_for_file = _file_md5(tc_filename)
+ self.assertEqual(md5_for_file, tc['md5'])
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(os.path.exists(info_json_fn))
- if 'md5' in tc:
- md5_for_file = _file_md5(tc_filename)
- self.assertEqual(md5_for_file, tc['md5'])
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
expect_info_dict(self, tc.get('info_dict', {}), info_dict)
finally:
try_rm_tcs_files()
+ if is_playlist and res_dict is not None:
+ # Remove all other files that may have been extracted if the
+ # extractor returns full results even with extract_flat
+ res_tcs = [{'info_dict': e} for e in res_dict['entries']]
+ try_rm_tcs_files(res_tcs)
return test_template
+++ /dev/null
-#!/usr/bin/env python
-# encoding: utf-8
-
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import (
- assertRegexpMatches,
- expect_info_dict,
- FakeYDL,
-)
-
-from youtube_dl.extractor import (
- AcademicEarthCourseIE,
- DailymotionPlaylistIE,
- DailymotionUserIE,
- VimeoChannelIE,
- VimeoUserIE,
- VimeoAlbumIE,
- VimeoGroupsIE,
- VineUserIE,
- UstreamChannelIE,
- SoundcloudSetIE,
- SoundcloudUserIE,
- SoundcloudPlaylistIE,
- TeacherTubeUserIE,
- LivestreamIE,
- LivestreamOriginalIE,
- NHLVideocenterIE,
- BambuserChannelIE,
- BandcampAlbumIE,
- SmotriCommunityIE,
- SmotriUserIE,
- IviCompilationIE,
- ImdbListIE,
- KhanAcademyIE,
- EveryonesMixtapeIE,
- RutubeChannelIE,
- RutubePersonIE,
- GoogleSearchIE,
- GenericIE,
- TEDIE,
- ToypicsUserIE,
- XTubeUserIE,
- InstagramUserIE,
- CSpanIE,
- AolIE,
-)
-
-
-class TestPlaylists(unittest.TestCase):
- def assertIsPlaylist(self, info):
- """Make sure the info has '_type' set to 'playlist'"""
- self.assertEqual(info['_type'], 'playlist')
-
- def test_dailymotion_playlist(self):
- dl = FakeYDL()
- ie = DailymotionPlaylistIE(dl)
- result = ie.extract('http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'SPORT')
- self.assertTrue(len(result['entries']) > 20)
-
- def test_dailymotion_user(self):
- dl = FakeYDL()
- ie = DailymotionUserIE(dl)
- result = ie.extract('https://www.dailymotion.com/user/nqtv')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'Rémi Gaillard')
- self.assertTrue(len(result['entries']) >= 100)
-
- def test_vimeo_channel(self):
- dl = FakeYDL()
- ie = VimeoChannelIE(dl)
- result = ie.extract('http://vimeo.com/channels/tributes')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'Vimeo Tributes')
- self.assertTrue(len(result['entries']) > 24)
-
- def test_vimeo_user(self):
- dl = FakeYDL()
- ie = VimeoUserIE(dl)
- result = ie.extract('http://vimeo.com/nkistudio/videos')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'Nki')
- self.assertTrue(len(result['entries']) > 65)
-
- def test_vimeo_album(self):
- dl = FakeYDL()
- ie = VimeoAlbumIE(dl)
- result = ie.extract('http://vimeo.com/album/2632481')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'Staff Favorites: November 2013')
- self.assertTrue(len(result['entries']) > 12)
-
- def test_vimeo_groups(self):
- dl = FakeYDL()
- ie = VimeoGroupsIE(dl)
- result = ie.extract('http://vimeo.com/groups/rolexawards')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'Rolex Awards for Enterprise')
- self.assertTrue(len(result['entries']) > 72)
-
- def test_vine_user(self):
- dl = FakeYDL()
- ie = VineUserIE(dl)
- result = ie.extract('https://vine.co/Visa')
- self.assertIsPlaylist(result)
- self.assertTrue(len(result['entries']) >= 47)
-
- def test_ustream_channel(self):
- dl = FakeYDL()
- ie = UstreamChannelIE(dl)
- result = ie.extract('http://www.ustream.tv/channel/channeljapan')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '10874166')
- self.assertTrue(len(result['entries']) >= 54)
-
- def test_soundcloud_set(self):
- dl = FakeYDL()
- ie = SoundcloudSetIE(dl)
- result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'The Royal Concept EP')
- self.assertTrue(len(result['entries']) >= 6)
-
- def test_soundcloud_user(self):
- dl = FakeYDL()
- ie = SoundcloudUserIE(dl)
- result = ie.extract('https://soundcloud.com/the-concept-band')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '9615865')
- self.assertTrue(len(result['entries']) >= 12)
-
- def test_soundcloud_likes(self):
- dl = FakeYDL()
- ie = SoundcloudUserIE(dl)
- result = ie.extract('https://soundcloud.com/the-concept-band/likes')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '9615865')
- self.assertTrue(len(result['entries']) >= 1)
-
- def test_soundcloud_playlist(self):
- dl = FakeYDL()
- ie = SoundcloudPlaylistIE(dl)
- result = ie.extract('http://api.soundcloud.com/playlists/4110309')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '4110309')
- self.assertEqual(result['title'], 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]')
- assertRegexpMatches(
- self, result['description'], r'TILT Brass - Bowery Poetry Club')
- self.assertEqual(len(result['entries']), 6)
-
- def test_livestream_event(self):
- dl = FakeYDL()
- ie = LivestreamIE(dl)
- result = ie.extract('http://new.livestream.com/tedx/cityenglish')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'TEDCity2.0 (English)')
- self.assertTrue(len(result['entries']) >= 4)
-
- def test_livestreamoriginal_folder(self):
- dl = FakeYDL()
- ie = LivestreamOriginalIE(dl)
- result = ie.extract('https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'a07bf706-d0e4-4e75-a747-b021d84f2fd3')
- self.assertTrue(len(result['entries']) >= 28)
-
- def test_nhl_videocenter(self):
- dl = FakeYDL()
- ie = NHLVideocenterIE(dl)
- result = ie.extract('http://video.canucks.nhl.com/videocenter/console?catid=999')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '999')
- self.assertEqual(result['title'], 'Highlights')
- self.assertEqual(len(result['entries']), 12)
-
- def test_bambuser_channel(self):
- dl = FakeYDL()
- ie = BambuserChannelIE(dl)
- result = ie.extract('http://bambuser.com/channel/pixelversity')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'pixelversity')
- self.assertTrue(len(result['entries']) >= 60)
-
- def test_bandcamp_album(self):
- dl = FakeYDL()
- ie = BandcampAlbumIE(dl)
- result = ie.extract('http://mpallante.bandcamp.com/album/nightmare-night-ep')
- self.assertIsPlaylist(result)
- self.assertEqual(result['title'], 'Nightmare Night EP')
- self.assertTrue(len(result['entries']) >= 4)
-
- def test_smotri_community(self):
- dl = FakeYDL()
- ie = SmotriCommunityIE(dl)
- result = ie.extract('http://smotri.com/community/video/kommuna')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'kommuna')
- self.assertEqual(result['title'], 'КПРФ')
- self.assertTrue(len(result['entries']) >= 4)
-
- def test_smotri_user(self):
- dl = FakeYDL()
- ie = SmotriUserIE(dl)
- result = ie.extract('http://smotri.com/user/inspector')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'inspector')
- self.assertEqual(result['title'], 'Inspector')
- self.assertTrue(len(result['entries']) >= 9)
-
- def test_AcademicEarthCourse(self):
- dl = FakeYDL()
- ie = AcademicEarthCourseIE(dl)
- result = ie.extract('http://academicearth.org/playlists/laws-of-nature/')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'laws-of-nature')
- self.assertEqual(result['title'], 'Laws of Nature')
- self.assertEqual(result['description'],u'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.')# u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.")
- self.assertEqual(len(result['entries']), 4)
-
- def test_ivi_compilation(self):
- dl = FakeYDL()
- ie = IviCompilationIE(dl)
- result = ie.extract('http://www.ivi.ru/watch/dvoe_iz_lartsa')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'dvoe_iz_lartsa')
- self.assertEqual(result['title'], 'Двое из ларца (2006 - 2008)')
- self.assertTrue(len(result['entries']) >= 24)
-
- def test_ivi_compilation_season(self):
- dl = FakeYDL()
- ie = IviCompilationIE(dl)
- result = ie.extract('http://www.ivi.ru/watch/dvoe_iz_lartsa/season1')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'dvoe_iz_lartsa/season1')
- self.assertEqual(result['title'], 'Двое из ларца (2006 - 2008) 1 сезон')
- self.assertTrue(len(result['entries']) >= 12)
-
- def test_imdb_list(self):
- dl = FakeYDL()
- ie = ImdbListIE(dl)
- result = ie.extract('http://www.imdb.com/list/JFs9NWw6XI0')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'JFs9NWw6XI0')
- self.assertEqual(result['title'], 'March 23, 2012 Releases')
- self.assertEqual(len(result['entries']), 7)
-
- def test_khanacademy_topic(self):
- dl = FakeYDL()
- ie = KhanAcademyIE(dl)
- result = ie.extract('https://www.khanacademy.org/math/applied-math/cryptography')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'cryptography')
- self.assertEqual(result['title'], 'Journey into cryptography')
- self.assertEqual(result['description'], 'How have humans protected their secret messages through history? What has changed today?')
- self.assertTrue(len(result['entries']) >= 3)
-
- def test_EveryonesMixtape(self):
- dl = FakeYDL()
- ie = EveryonesMixtapeIE(dl)
- result = ie.extract('http://everyonesmixtape.com/#/mix/m7m0jJAbMQi')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'm7m0jJAbMQi')
- self.assertEqual(result['title'], 'Driving')
- self.assertEqual(len(result['entries']), 24)
-
- def test_rutube_channel(self):
- dl = FakeYDL()
- ie = RutubeChannelIE(dl)
- result = ie.extract('http://rutube.ru/tags/video/1800/')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '1800')
- self.assertTrue(len(result['entries']) >= 68)
-
- def test_rutube_person(self):
- dl = FakeYDL()
- ie = RutubePersonIE(dl)
- result = ie.extract('http://rutube.ru/video/person/313878/')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '313878')
- self.assertTrue(len(result['entries']) >= 37)
-
- def test_multiple_brightcove_videos(self):
- # https://github.com/rg3/youtube-dl/issues/2283
- dl = FakeYDL()
- ie = GenericIE(dl)
- result = ie.extract('http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'always-never-nuclear-command-and-control')
- self.assertEqual(result['title'], 'Always/Never: A Little-Seen Movie About Nuclear Command and Control : The New Yorker')
- self.assertEqual(len(result['entries']), 3)
-
- def test_GoogleSearch(self):
- dl = FakeYDL()
- ie = GoogleSearchIE(dl)
- result = ie.extract('gvsearch15:python language')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'python language')
- self.assertEqual(result['title'], 'python language')
- self.assertEqual(len(result['entries']), 15)
-
- def test_generic_rss_feed(self):
- dl = FakeYDL()
- ie = GenericIE(dl)
- result = ie.extract('http://phihag.de/2014/youtube-dl/rss.xml')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'http://phihag.de/2014/youtube-dl/rss.xml')
- self.assertEqual(result['title'], 'Zero Punctuation')
- self.assertTrue(len(result['entries']) > 10)
-
- def test_ted_playlist(self):
- dl = FakeYDL()
- ie = TEDIE(dl)
- result = ie.extract('http://www.ted.com/playlists/who_are_the_hackers')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '10')
- self.assertEqual(result['title'], 'Who are the hackers?')
- self.assertTrue(len(result['entries']) >= 6)
-
- def test_toypics_user(self):
- dl = FakeYDL()
- ie = ToypicsUserIE(dl)
- result = ie.extract('http://videos.toypics.net/Mikey')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'Mikey')
- self.assertTrue(len(result['entries']) >= 17)
-
- def test_xtube_user(self):
- dl = FakeYDL()
- ie = XTubeUserIE(dl)
- result = ie.extract('http://www.xtube.com/community/profile.php?user=greenshowers')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'greenshowers')
- self.assertTrue(len(result['entries']) >= 155)
-
- def test_InstagramUser(self):
- dl = FakeYDL()
- ie = InstagramUserIE(dl)
- result = ie.extract('http://instagram.com/porsche')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'porsche')
- self.assertTrue(len(result['entries']) >= 2)
- test_video = next(
- e for e in result['entries']
- if e['id'] == '614605558512799803_462752227')
- dl.add_default_extra_info(test_video, ie, '(irrelevant URL)')
- dl.process_video_result(test_video, download=False)
- EXPECTED = {
- 'id': '614605558512799803_462752227',
- 'ext': 'mp4',
- 'title': '#Porsche Intelligent Performance.',
- 'thumbnail': 're:^https?://.*\.jpg',
- 'uploader': 'Porsche',
- 'uploader_id': 'porsche',
- 'timestamp': 1387486713,
- 'upload_date': '20131219',
- }
- expect_info_dict(self, EXPECTED, test_video)
-
- def test_CSpan_playlist(self):
- dl = FakeYDL()
- ie = CSpanIE(dl)
- result = ie.extract(
- 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '342759')
- self.assertEqual(
- result['title'], 'General Motors Ignition Switch Recall')
- whole_duration = sum(e['duration'] for e in result['entries'])
- self.assertEqual(whole_duration, 14855)
-
- def test_aol_playlist(self):
- dl = FakeYDL()
- ie = AolIE(dl)
- result = ie.extract(
- 'http://on.aol.com/playlist/brace-yourself---todays-weirdest-news-152147?icid=OnHomepageC4_Omg_Img#_videoid=518184316')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], '152147')
- self.assertEqual(
- result['title'], 'Brace Yourself - Today\'s Weirdest News')
- self.assertTrue(len(result['entries']) >= 10)
-
- def test_TeacherTubeUser(self):
- dl = FakeYDL()
- ie = TeacherTubeUserIE(dl)
- result = ie.extract('http://www.teachertube.com/user/profile/rbhagwati2')
- self.assertIsPlaylist(result)
- self.assertEqual(result['id'], 'rbhagwati2')
- self.assertTrue(len(result['entries']) >= 179)
-
-if __name__ == '__main__':
- unittest.main()
--- /dev/null
+#!/usr/bin/env python
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+import errno
+import io
+import json
+import re
+import subprocess
+
+from youtube_dl.swfinterp import SWFInterpreter
+
+
+TEST_DIR = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), 'swftests')
+
+
+class TestSWFInterpreter(unittest.TestCase):
+ pass
+
+
+def _make_testfunc(testfile):
+ m = re.match(r'^(.*)\.(as)$', testfile)
+ if not m:
+ return
+ test_id = m.group(1)
+
+ def test_func(self):
+ as_file = os.path.join(TEST_DIR, testfile)
+ swf_file = os.path.join(TEST_DIR, test_id + '.swf')
+ if ((not os.path.exists(swf_file))
+ or os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
+ # Recompile
+ try:
+ subprocess.check_call(['mxmlc', '-output', swf_file, as_file])
+ except OSError as ose:
+ if ose.errno == errno.ENOENT:
+ print('mxmlc not found! Skipping test.')
+ return
+ raise
+
+ with open(swf_file, 'rb') as swf_f:
+ swf_content = swf_f.read()
+ swfi = SWFInterpreter(swf_content)
+
+ with io.open(as_file, 'r', encoding='utf-8') as as_f:
+ as_content = as_f.read()
+
+ def _find_spec(key):
+ m = re.search(
+ r'(?m)^//\s*%s:\s*(.*?)\n' % re.escape(key), as_content)
+ if not m:
+ raise ValueError('Cannot find %s in %s' % (key, testfile))
+ return json.loads(m.group(1))
+
+ input_args = _find_spec('input')
+ output = _find_spec('output')
+
+ swf_class = swfi.extract_class(test_id)
+ func = swfi.extract_function(swf_class, 'main')
+ res = func(input_args)
+ self.assertEqual(res, output)
+
+ test_func.__name__ = str('test_swf_' + test_id)
+ setattr(TestSWFInterpreter, test_func.__name__, test_func)
+
+
+for testfile in os.listdir(TEST_DIR):
+ _make_testfunc(testfile)
+
+if __name__ == '__main__':
+ unittest.main()
#!/usr/bin/env python
# coding: utf-8
+from __future__ import unicode_literals
+
# Allow direct execution
import os
import sys
import json
import xml.etree.ElementTree
-#from youtube_dl.utils import htmlentity_transform
from youtube_dl.utils import (
DateRange,
encodeFilename,
uppercase_escape,
)
-if sys.version_info < (3, 0):
- _compat_str = lambda b: b.decode('unicode-escape')
-else:
- _compat_str = lambda s: s
-
class TestUtil(unittest.TestCase):
def test_timeconvert(self):
self.assertEqual('this - that', sanitize_filename('this: that'))
self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
- aumlaut = _compat_str('\xe4')
+ aumlaut = 'ä'
self.assertEqual(sanitize_filename(aumlaut), aumlaut)
- tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
+ tests = '\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430'
self.assertEqual(sanitize_filename(tests), tests)
forbidden = '"\0\\/'
self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
- tests = _compat_str('a\xe4b\u4e2d\u56fd\u7684c')
+ tests = 'a\xe4b\u4e2d\u56fd\u7684c'
self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
- self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '') # No empty filename
+ self.assertTrue(sanitize_filename('\xf6', restricted=True) != '') # No empty filename
forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#'
for fc in forbidden:
self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
# Handle a common case more neatly
- self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
- self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
+ self.assertEqual(sanitize_filename('\u5927\u58f0\u5e26 - Song', restricted=True), 'Song')
+ self.assertEqual(sanitize_filename('\u603b\u7edf: Speech', restricted=True), 'Speech')
# .. but make sure the file name is never empty
self.assertTrue(sanitize_filename('-', restricted=True) != '')
self.assertTrue(sanitize_filename(':', restricted=True) != '')
self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1])
def test_unescape_html(self):
- self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
+ self.assertEqual(unescapeHTML('%20;'), '%20;')
+ self.assertEqual(
+ unescapeHTML('é'), 'é')
def test_daterange(self):
_20century = DateRange("19000101","20000101")
self.assertEqual(unified_strdate('1968-12-10'), '19681210')
def test_find_xpath_attr(self):
- testxml = u'''<root>
+ testxml = '''<root>
<node/>
<node x="a"/>
<node x="a" y="c" />
self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
def test_meta_parser(self):
- testhtml = u'''
+ testhtml = '''
<head>
<meta name="description" content="foo & bar">
<meta content='Plato' name='author'/>
</head>
'''
get_meta = lambda name: get_meta_content(name, testhtml)
- self.assertEqual(get_meta('description'), u'foo & bar')
+ self.assertEqual(get_meta('description'), 'foo & bar')
self.assertEqual(get_meta('author'), 'Plato')
def test_xpath_with_ns(self):
- testxml = u'''<root xmlns:media="http://example.com/">
+ testxml = '''<root xmlns:media="http://example.com/">
<media:song>
<media:author>The Author</media:author>
<url>http://server.com/download.mp3</url>
doc = xml.etree.ElementTree.fromstring(testxml)
find = lambda p: doc.find(xpath_with_ns(p, {'media': 'http://example.com/'}))
self.assertTrue(find('media:song') is not None)
- self.assertEqual(find('media:song/media:author').text, u'The Author')
- self.assertEqual(find('media:song/url').text, u'http://server.com/download.mp3')
+ self.assertEqual(find('media:song/media:author').text, 'The Author')
+ self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')
def test_smuggle_url(self):
data = {u"ö": u"ö", u"abc": [3]}
self.assertEqual(res_data, None)
def test_shell_quote(self):
- args = ['ffmpeg', '-i', encodeFilename(u'ñ€ß\'.mp4')]
- self.assertEqual(shell_quote(args), u"""ffmpeg -i 'ñ€ß'"'"'.mp4'""")
+ args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
+ self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")
def test_str_to_int(self):
self.assertEqual(str_to_int('123,456'), 123456)
self.assertEqual(str_to_int('123.456'), 123456)
def test_url_basename(self):
- self.assertEqual(url_basename(u'http://foo.de/'), u'')
- self.assertEqual(url_basename(u'http://foo.de/bar/baz'), u'baz')
- self.assertEqual(url_basename(u'http://foo.de/bar/baz?x=y'), u'baz')
- self.assertEqual(url_basename(u'http://foo.de/bar/baz#x=y'), u'baz')
- self.assertEqual(url_basename(u'http://foo.de/bar/baz/'), u'baz')
+ self.assertEqual(url_basename('http://foo.de/'), '')
+ self.assertEqual(url_basename('http://foo.de/bar/baz'), 'baz')
+ self.assertEqual(url_basename('http://foo.de/bar/baz?x=y'), 'baz')
+ self.assertEqual(url_basename('http://foo.de/bar/baz#x=y'), 'baz')
+ self.assertEqual(url_basename('http://foo.de/bar/baz/'), 'baz')
self.assertEqual(
- url_basename(u'http://media.w3.org/2010/05/sintel/trailer.mp4'),
- u'trailer.mp4')
+ url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'),
+ 'trailer.mp4')
def test_parse_duration(self):
self.assertEqual(parse_duration(None), None)
self.assertEqual(parse_duration('00:01:01'), 61)
self.assertEqual(parse_duration('x:y'), None)
self.assertEqual(parse_duration('3h11m53s'), 11513)
+ self.assertEqual(parse_duration('3h 11m 53s'), 11513)
+ self.assertEqual(parse_duration('3 hours 11 minutes 53 seconds'), 11513)
+ self.assertEqual(parse_duration('3 hours 11 mins 53 secs'), 11513)
self.assertEqual(parse_duration('62m45s'), 3765)
self.assertEqual(parse_duration('6m59s'), 419)
self.assertEqual(parse_duration('49s'), 49)
self.assertEqual(parse_duration('0h0m0s'), 0)
self.assertEqual(parse_duration('0m0s'), 0)
self.assertEqual(parse_duration('0s'), 0)
+ self.assertEqual(parse_duration('01:02:03.05'), 3723.05)
def test_fix_xml_ampersands(self):
self.assertEqual(
testPL(5, 2, (20, 99), [])
def test_struct_unpack(self):
- self.assertEqual(struct_unpack(u'!B', b'\x00'), (0,))
+ self.assertEqual(struct_unpack('!B', b'\x00'), (0,))
def test_read_batch_urls(self):
- f = io.StringIO(u'''\xef\xbb\xbf foo
+ f = io.StringIO('''\xef\xbb\xbf foo
bar\r
baz
# More after this line\r
; or after this
bam''')
- self.assertEqual(read_batch_urls(f), [u'foo', u'bar', u'baz', u'bam'])
+ self.assertEqual(read_batch_urls(f), ['foo', 'bar', 'baz', 'bam'])
def test_urlencode_postdata(self):
data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
d = json.loads(stripped)
self.assertEqual(d, [{"id": "532cb", "x": 3}])
- def test_uppercase_escpae(self):
- self.assertEqual(uppercase_escape(u'aä'), u'aä')
- self.assertEqual(uppercase_escape(u'\\U0001d550'), u'𝕐')
+ def test_uppercase_escape(self):
+ self.assertEqual(uppercase_escape('aä'), 'aä')
+ self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
if __name__ == '__main__':
unittest.main()
#!/usr/bin/env python
+from __future__ import unicode_literals
+
# Allow direct execution
import os
import sys
_TESTS = [
(
- u'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
- u'js',
+ 'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
+ 'js',
86,
- u'>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
+ '>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
),
(
- u'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
- u'js',
+ 'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
+ 'js',
85,
- u'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
+ '3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
),
(
- u'https://s.ytimg.com/yts/jsbin/html5player-vfle-mVwz.js',
- u'js',
+ 'https://s.ytimg.com/yts/jsbin/html5player-vfle-mVwz.js',
+ 'js',
90,
- u']\\[@?>=<;:/.-,+*)(\'&%$#"hZYXWVUTSRQPONMLKJIHGFEDCBAzyxwvutsrqponmlkjiagfedcb39876',
+ ']\\[@?>=<;:/.-,+*)(\'&%$#"hZYXWVUTSRQPONMLKJIHGFEDCBAzyxwvutsrqponmlkjiagfedcb39876',
+ ),
+ (
+ 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl0Cbn9e.js',
+ 'js',
+ 84,
+ 'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVW@YZ!"#$%&\'()*+,-./:;<=',
+ ),
+ (
+ 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js',
+ 'js',
+ '2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
+ 'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
),
(
- u'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl0Cbn9e.js',
- u'js',
+ 'http://s.ytimg.com/yts/swfbin/player-vfl5vIhK2/watch_as3.swf',
+ 'swf',
+ 86,
+ 'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVWXY\\!"#$%&\'()*+,-./:;<=>?'
+ ),
+ (
+ 'http://s.ytimg.com/yts/swfbin/player-vflmDyk47/watch_as3.swf',
+ 'swf',
+ 'F375F75BF2AFDAAF2666E43868D46816F83F13E81C46.3725A8218E446A0DECD33F79DC282994D6AA92C92C9',
+ '9C29AA6D499282CD97F33DCED0A644E8128A5273.64C18E31F38361864D86834E6662FAADFA2FB57F'
+ ),
+ (
+ 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
+ 'js',
84,
- u'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVW@YZ!"#$%&\'()*+,-./:;<=',
+ '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>'
),
(
- u'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js',
- u'js',
- u'2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
- u'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
+ 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js',
+ 'js',
+ 83,
+ '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F'
),
+ (
+ 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js',
+ 'js',
+ '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
+ '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
+ )
]
def make_tfunc(url, stype, sig_input, expected_sig):
- basename = url.rpartition('/')[2]
- m = re.match(r'.*-([a-zA-Z0-9_-]+)\.[a-z]+$', basename)
- assert m, '%r should follow URL format' % basename
+ m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
+ assert m, '%r should follow URL format' % url
test_id = m.group(1)
def test_func(self):
+ basename = 'player-%s.%s' % (test_id, stype)
fn = os.path.join(self.TESTDATA_DIR, basename)
if not os.path.exists(fn):
+++ /dev/null
-# Legacy file for backwards compatibility, use youtube_dl.downloader instead!
-from .downloader import FileDownloader as RealFileDownloader
-from .downloader import get_suitable_downloader
-
-
-# This class reproduces the old behaviour of FileDownloader
-class FileDownloader(RealFileDownloader):
- def _do_download(self, filename, info_dict):
- real_fd = get_suitable_downloader(info_dict)(self.ydl, self.params)
- for ph in self._progress_hooks:
- real_fd.add_progress_hook(ph)
- return real_fd.download(filename, info_dict)
YoutubeDLHandler,
prepend_extension,
)
+from .cache import Cache
from .extractor import get_info_extractor, gen_extractors
from .downloader import get_suitable_downloader
from .postprocessor import FFmpegMergerPP
daterange: A DateRange object, download only if the upload_date is in the range.
skip_download: Skip the actual download of the video file
cachedir: Location of the cache files in the filesystem.
- None to disable filesystem cache.
+ False to disable filesystem cache.
noplaylist: Download single video instead of a playlist if in doubt.
age_limit: An integer representing the user's age in years.
Unsuitable videos for the given age are skipped.
default_search: Prepend this string if an input url is not valid.
'auto' for elaborate guessing
encoding: Use this encoding instead of the system-specified.
+ extract_flat: Do not resolve URLs, return the immediate result.
The following parameters are not used by YoutubeDL itself, they are used by
the FileDownloader:
The following options are used by the post processors:
prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
otherwise prefer avconv.
+ exec_cmd: Arbitrary command to run after downloading
"""
params = None
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self._err_file = sys.stderr
self.params = params
+ self.cache = Cache(self)
if params.get('bidi_workaround', False):
try:
return message
assert hasattr(self, '_output_process')
- assert type(message) == type('')
+ assert isinstance(message, compat_str)
line_count = message.count('\n') + 1
self._output_process.stdin.write((message + '\n').encode('utf-8'))
self._output_process.stdin.flush()
def to_stderr(self, message):
"""Print message to stderr."""
- assert type(message) == type('')
+ assert isinstance(message, compat_str)
if self.params.get('logger'):
self.params['logger'].error(message)
else:
autonumber_templ = '%0' + str(autonumber_size) + 'd'
template_dict['autonumber'] = autonumber_templ % self._num_downloads
if template_dict.get('playlist_index') is not None:
- template_dict['playlist_index'] = '%05d' % template_dict['playlist_index']
+ template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
if template_dict.get('resolution') is None:
if template_dict.get('width') and template_dict.get('height'):
template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
age_limit = self.params.get('age_limit')
if age_limit is not None:
- if age_limit < info_dict.get('age_limit', 0):
+ actual_age_limit = info_dict.get('age_limit')
+ if actual_age_limit is None:
+ actual_age_limit = 0
+ if age_limit < actual_age_limit:
return 'Skipping "' + title + '" because it is age restricted'
if self.in_download_archive(info_dict):
return '%s has already been recorded in archive' % video_title
Returns the resolved ie_result.
"""
- result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
+ result_type = ie_result.get('_type', 'video')
+
+ if self.params.get('extract_flat', False):
+ if result_type in ('url', 'url_transparent'):
+ return ie_result
+
if result_type == 'video':
self.add_extra_info(ie_result, extra_info)
return self.process_video_result(ie_result, download=download)
for i, entry in enumerate(entries, 1):
self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
extra = {
+ 'n_entries': n_entries,
'playlist': playlist,
'playlist_index': i + playliststart,
'extractor': ie_result['extractor'],
# Keep for backwards compatibility
info_dict['stitle'] = info_dict['title']
- if not 'format' in info_dict:
+ if 'format' not in info_dict:
info_dict['format'] = info_dict['ext']
reason = self._match_entry(info_dict)
if info_dict.get('requested_formats') is not None:
downloaded = []
success = True
- merger = FFmpegMergerPP(self)
+ merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
if not merger._get_executable():
postprocessors = []
self.report_warning('You have requested multiple '
if res:
res += ', '
res += format_bytes(fdict['filesize'])
+ elif fdict.get('filesize_approx') is not None:
+ if res:
+ res += ', '
+ res += '~' + format_bytes(fdict['filesize_approx'])
return res
def list_formats(self, info_dict):
if not self.params.get('verbose'):
return
- write_string(
+ if type('') is not compat_str:
+ # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
+ self.report_warning(
+ 'Your Python is broken! Update to a newer and supported version')
+
+ encoding_str = (
'[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
locale.getpreferredencoding(),
sys.getfilesystemencoding(),
sys.stdout.encoding,
- self.get_encoding()),
- encoding=None
- )
+ self.get_encoding()))
+ write_string(encoding_str, encoding=None)
self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
try:
'Adam Malcontenti-Wilson',
'Tobias Bell',
'Naglis Jonaitis',
+ 'Charles Chen',
+ 'Hassaan Ali',
+ 'Dobrosław Żybort',
+ 'David Fabijan',
+ 'Sebastian Haas',
+ 'Alexander Kirk',
+ 'Erik Johnson',
+ 'Keith Beckman',
+ 'Ole Ernst',
+ 'Aaron McDaniel (mcd1992)',
+ 'Magnus Kolstad',
)
__license__ = 'Public Domain'
import codecs
import io
-import locale
import optparse
import os
import random
-import re
import shlex
import sys
decodeOption,
get_term_width,
DownloadError,
- get_cachedir,
MaxDownloadsReached,
preferredencoding,
read_batch_urls,
write_string,
)
from .update import update_self
-from .FileDownloader import (
+from .downloader import (
FileDownloader,
)
from .extractor import gen_extractors
FFmpegExtractAudioPP,
FFmpegEmbedSubtitlePP,
XAttrMetadataPP,
+ ExecAfterDownloadPP,
)
downloader = optparse.OptionGroup(parser, 'Download Options')
postproc = optparse.OptionGroup(parser, 'Post-processing Options')
filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
+ workarounds = optparse.OptionGroup(parser, 'Workarounds')
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
general.add_option('-h', '--help',
general.add_option('--dump-user-agent',
action='store_true', dest='dump_user_agent',
help='display the current browser identification', default=False)
- general.add_option('--user-agent',
- dest='user_agent', help='specify a custom user agent', metavar='UA')
- general.add_option('--referer',
- dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
- metavar='REF', default=None)
- general.add_option('--add-header',
- dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
- metavar='FIELD:VALUE')
general.add_option('--list-extractors',
action='store_true', dest='list_extractors',
help='List all supported extractors and the URLs they would handle', default=False)
general.add_option(
'--proxy', dest='proxy', default=None, metavar='URL',
help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
- general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
- general.add_option(
- '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
- help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
- general.add_option(
- '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
- help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
- general.add_option(
- '--no-cache-dir', action='store_const', const=None, dest='cachedir',
- help='Disable filesystem caching')
general.add_option(
'--socket-timeout', dest='socket_timeout',
type=float, default=None, help=u'Time to wait before giving up, in seconds')
- general.add_option(
- '--bidi-workaround', dest='bidi_workaround', action='store_true',
- help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
general.add_option(
'--default-search',
dest='default_search', metavar='PREFIX',
- help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess. The default value "error" just throws an error.')
+ help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
general.add_option(
'--ignore-config',
action='store_true',
help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
- general.add_option(
- '--encoding', dest='encoding', metavar='ENCODING',
- help='Force the specified encoding (experimental)')
selection.add_option(
'--playlist-start',
dest='username', metavar='USERNAME', help='account username')
authentication.add_option('-p', '--password',
dest='password', metavar='PASSWORD', help='account password')
+ authentication.add_option('-2', '--twofactor',
+ dest='twofactor', metavar='TWOFACTOR', help='two-factor auth code')
authentication.add_option('-n', '--netrc',
action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
authentication.add_option('--video-password',
help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
+ workarounds.add_option(
+ '--encoding', dest='encoding', metavar='ENCODING',
+ help='Force the specified encoding (experimental)')
+ workarounds.add_option(
+ '--no-check-certificate', action='store_true',
+ dest='no_check_certificate', default=False,
+ help='Suppress HTTPS certificate validation.')
+ workarounds.add_option(
+ '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
+ help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
+ workarounds.add_option(
+ '--user-agent', metavar='UA',
+ dest='user_agent', help='specify a custom user agent')
+ workarounds.add_option(
+ '--referer', metavar='REF',
+ dest='referer', default=None,
+ help='specify a custom referer, use if the video access is restricted to one domain',
+ )
+ workarounds.add_option(
+ '--add-header', metavar='FIELD:VALUE',
+ dest='headers', action='append',
+ help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
+ )
+ workarounds.add_option(
+ '--bidi-workaround', dest='bidi_workaround', action='store_true',
+ help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
+
verbosity.add_option('-q', '--quiet',
action='store_true', dest='quiet', help='activates quiet mode', default=False)
verbosity.add_option(
help='Display sent and read HTTP traffic')
- filesystem.add_option('-t', '--title',
- action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
+ filesystem.add_option('-a', '--batch-file',
+ dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
filesystem.add_option('--id',
action='store_true', dest='useid', help='use only video ID in file name', default=False)
- filesystem.add_option('-l', '--literal',
- action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
filesystem.add_option('-A', '--auto-number',
action='store_true', dest='autonumber',
help='number downloaded files starting from 00000', default=False)
filesystem.add_option('--restrict-filenames',
action='store_true', dest='restrictfilenames',
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
- filesystem.add_option('-a', '--batch-file',
- dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
- filesystem.add_option('--load-info',
- dest='load_info_filename', metavar='FILE',
- help='json file containing the video information (created with the "--write-json" option)')
+ filesystem.add_option('-t', '--title',
+ action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
+ filesystem.add_option('-l', '--literal',
+ action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
filesystem.add_option('-w', '--no-overwrites',
action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
filesystem.add_option('-c', '--continue',
filesystem.add_option('--no-continue',
action='store_false', dest='continue_dl',
help='do not resume partially downloaded files (restart from beginning)')
- filesystem.add_option('--cookies',
- dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
filesystem.add_option('--no-part',
action='store_true', dest='nopart', help='do not use .part files', default=False)
filesystem.add_option('--no-mtime',
filesystem.add_option('--write-thumbnail',
action='store_true', dest='writethumbnail',
help='write thumbnail image to disk', default=False)
+ filesystem.add_option('--load-info',
+ dest='load_info_filename', metavar='FILE',
+ help='json file containing the video information (created with the "--write-json" option)')
+ filesystem.add_option('--cookies',
+ dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
+ filesystem.add_option(
+ '--cache-dir', dest='cachedir', default=None, metavar='DIR',
+ help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
+ filesystem.add_option(
+ '--no-cache-dir', action='store_const', const=False, dest='cachedir',
+ help='Disable filesystem caching')
+ filesystem.add_option(
+ '--rm-cache-dir', action='store_true', dest='rm_cachedir',
+ help='Delete all filesystem cache files')
postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
help='Prefer avconv over ffmpeg for running the postprocessors (default)')
postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
help='Prefer ffmpeg over avconv for running the postprocessors')
-
+ postproc.add_option(
+ '--exec', metavar='CMD', dest='exec_cmd',
+ help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'' )
parser.add_option_group(general)
parser.add_option_group(selection)
parser.add_option_group(downloader)
parser.add_option_group(filesystem)
parser.add_option_group(verbosity)
+ parser.add_option_group(workarounds)
parser.add_option_group(video_format)
parser.add_option_group(subtitles)
parser.add_option_group(authentication)
if desc is False:
continue
if hasattr(ie, 'SEARCH_KEY'):
- _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
+ _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
_COUNTS = (u'', u'5', u'10', u'all')
desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
compat_print(desc)
date = DateRange.day(opts.date)
else:
date = DateRange(opts.dateafter, opts.datebefore)
- if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
+ if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
# Do not download videos when there are audio-only formats
'usenetrc': opts.usenetrc,
'username': opts.username,
'password': opts.password,
+ 'twofactor': opts.twofactor,
'videopassword': opts.videopassword,
'quiet': (opts.quiet or any_printing),
'no_warnings': opts.no_warnings,
'default_search': opts.default_search,
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
'encoding': opts.encoding,
+ 'exec_cmd': opts.exec_cmd,
}
with YoutubeDL(ydl_opts) as ydl:
ydl.add_post_processor(FFmpegAudioFixPP())
ydl.add_post_processor(AtomicParsleyPP())
+
+ # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
+ # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
+ if opts.exec_cmd:
+ ydl.add_post_processor(ExecAfterDownloadPP(
+ verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd))
+
# Update version
if opts.update_self:
update_self(ydl.to_screen, opts.verbose)
+ # Remove cache dir
+ if opts.rm_cachedir:
+ ydl.cache.remove()
+
# Maybe do nothing
if (len(all_urls) < 1) and (opts.load_info_filename is None):
- if not opts.update_self:
+ if not (opts.update_self or opts.rm_cachedir):
parser.error(u'you must provide at least one URL')
else:
sys.exit()
--- /dev/null
+from __future__ import unicode_literals
+
+import errno
+import io
+import json
+import os
+import re
+import shutil
+import traceback
+
+from .utils import (
+ write_json_file,
+)
+
+
+class Cache(object):
+ def __init__(self, ydl):
+ self._ydl = ydl
+
+ def _get_root_dir(self):
+ res = self._ydl.params.get('cachedir')
+ if res is None:
+ cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache')
+ res = os.path.join(cache_root, 'youtube-dl')
+ return os.path.expanduser(res)
+
+ def _get_cache_fn(self, section, key, dtype):
+ assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
+ 'invalid section %r' % section
+ assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
+ return os.path.join(
+ self._get_root_dir(), section, '%s.%s' % (key, dtype))
+
+ @property
+ def enabled(self):
+ return self._ydl.params.get('cachedir') is not False
+
+ def store(self, section, key, data, dtype='json'):
+ assert dtype in ('json',)
+
+ if not self.enabled:
+ return
+
+ fn = self._get_cache_fn(section, key, dtype)
+ try:
+ try:
+ os.makedirs(os.path.dirname(fn))
+ except OSError as ose:
+ if ose.errno != errno.EEXIST:
+ raise
+ write_json_file(data, fn)
+ except Exception:
+ tb = traceback.format_exc()
+ self._ydl.report_warning(
+ 'Writing cache to %r failed: %s' % (fn, tb))
+
+ def load(self, section, key, dtype='json', default=None):
+ assert dtype in ('json',)
+
+ if not self.enabled:
+ return default
+
+ cache_fn = self._get_cache_fn(section, key, dtype)
+ try:
+ try:
+ with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
+ return json.load(cachef)
+ except ValueError:
+ try:
+ file_size = os.path.getsize(cache_fn)
+ except (OSError, IOError) as oe:
+ file_size = str(oe)
+ self._ydl.report_warning(
+ 'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
+ except IOError:
+ pass # No cache available
+
+ return default
+
+ def remove(self):
+ if not self.enabled:
+ self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
+ return
+
+ cachedir = self._get_root_dir()
+ if not any((term in cachedir) for term in ('cache', 'tmp')):
+ raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)
+
+ self._ydl.to_screen(
+ 'Removing cache dir %s .' % cachedir, skip_eol=True)
+ if os.path.exists(cachedir):
+ self._ydl.to_screen('.', skip_eol=True)
+ shutil.rmtree(cachedir)
+ self._ydl.to_screen('.')
def real_download(self, filename, info_dict):
"""Real download process. Redefine in subclasses."""
- raise NotImplementedError(u'This method must be implemented by sublcasses')
+ raise NotImplementedError(u'This method must be implemented by subclasses')
def _hook_progress(self, status):
for ph in self._progress_hooks:
def real_download(self, filename, info_dict):
man_url = info_dict['url']
+ requested_bitrate = info_dict.get('tbr')
self.to_screen('[download] Downloading f4m manifest')
manifest = self.ydl.urlopen(man_url).read()
self.report_destination(filename)
doc = etree.fromstring(manifest)
formats = [(int(f.attrib.get('bitrate', -1)), f) for f in doc.findall(_add_ns('media'))]
- formats = sorted(formats, key=lambda f: f[0])
- rate, media = formats[-1]
+ if requested_bitrate is None:
+ # get the best format
+ formats = sorted(formats, key=lambda f: f[0])
+ rate, media = formats[-1]
+ else:
+ rate, media = list(filter(
+ lambda f: int(f[0]) == requested_bitrate, formats))[0]
+
base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
bootstrap = base64.b64decode(doc.find(_add_ns('bootstrapInfo')).text)
metadata = base64.b64decode(media.find(_add_ns('metadata')).text)
from .common import FileDownloader
from ..utils import (
+ check_executable,
encodeFilename,
)
encodeFilename(tmpfilename, for_subprocess=True)]
for program in ['avconv', 'ffmpeg']:
- try:
- subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+ if check_executable(program, ['-version']):
break
- except (OSError, IOError):
- pass
else:
self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
+ return False
cmd = [program] + args
retval = subprocess.call(cmd)
return True
else:
self.to_stderr(u"\n")
- self.report_error(u'ffmpeg exited with code %d' % retval)
+ self.report_error(u'%s exited with code %d' % (program, retval))
return False
headers['Youtubedl-user-agent'] = info_dict['user_agent']
if 'http_referer' in info_dict:
headers['Referer'] = info_dict['http_referer']
- basic_request = compat_urllib_request.Request(url, None, headers)
- request = compat_urllib_request.Request(url, None, headers)
+ add_headers = info_dict.get('http_headers')
+ if add_headers:
+ headers.update(add_headers)
+ data = info_dict.get('http_post_data')
+ http_method = info_dict.get('http_method')
+ basic_request = compat_urllib_request.Request(url, data, headers)
+ request = compat_urllib_request.Request(url, data, headers)
+ if http_method is not None:
+ basic_request.get_method = lambda: http_method
+ request.get_method = lambda: http_method
is_test = self.params.get('test', False)
self.to_stderr(u"\n")
self.report_error(u'Did not get any data blocks')
return False
- stream.close()
+ if tmpfilename != u'-':
+ stream.close()
self.report_finish(data_len_str, (time.time() - start))
if data_len is not None and byte_counter != data_len:
raise ContentTooShortError(byte_counter, int(data_len))
from .common import FileDownloader
from ..utils import (
+ check_executable,
+ compat_str,
encodeFilename,
format_bytes,
- compat_str,
)
test = self.params.get('test', False)
# Check for rtmpdump first
- try:
- subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
- except (OSError, IOError):
+ if not check_executable('rtmpdump', ['-h']):
self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install it.')
return False
+from .abc import ABCIE
from .academicearth import AcademicEarthCourseIE
from .addanime import AddAnimeIE
+from .adultswim import AdultSwimIE
from .aftonbladet import AftonbladetIE
from .anitube import AnitubeIE
+from .anysex import AnySexIE
from .aol import AolIE
from .allocine import AllocineIE
from .aparat import AparatIE
from .appletrailers import AppleTrailersIE
from .archiveorg import ArchiveOrgIE
-from .ard import ARDIE
+from .ard import ARDIE, ARDMediathekIE
from .arte import (
ArteTvIE,
ArteTVPlus7IE,
from .bambuser import BambuserIE, BambuserChannelIE
from .bandcamp import BandcampIE, BandcampAlbumIE
from .bbccouk import BBCCoUkIE
+from .beeg import BeegIE
from .bilibili import BiliBiliIE
from .blinkx import BlinkxIE
from .bliptv import BlipTVIE, BlipTVUserIE
from .collegehumor import CollegeHumorIE
from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
from .condenast import CondeNastIE
+from .cracked import CrackedIE
from .criterion import CriterionIE
from .crunchyroll import CrunchyrollIE
from .cspan import CSpanIE
DailymotionUserIE,
)
from .daum import DaumIE
+from .dbtv import DBTVIE
+from .dfb import DFBIE
from .dotsub import DotsubIE
from .dreisat import DreiSatIE
+from .drtuber import DrTuberIE
from .drtv import DRTVIE
+from .dump import DumpIE
from .defense import DefenseGouvFrIE
from .discovery import DiscoveryIE
from .divxstage import DivxStageIE
from .ehow import EHowIE
from .eighttracks import EightTracksIE
from .eitb import EitbIE
+from .ellentv import (
+ EllenTVIE,
+ EllenTVClipsIE,
+)
from .elpais import ElPaisIE
-from .empflix import EmpflixIE
+from .empflix import EMPFlixIE
from .engadget import EngadgetIE
+from .eporner import EpornerIE
from .escapist import EscapistIE
from .everyonesmixtape import EveryonesMixtapeIE
from .exfm import ExfmIE
+from .expotv import ExpoTVIE
from .extremetube import ExtremeTubeIE
from .facebook import FacebookIE
from .faz import FazIE
from .freespeech import FreespeechIE
from .funnyordie import FunnyOrDieIE
from .gamekings import GamekingsIE
-from .gameone import GameOneIE
+from .gameone import (
+ GameOneIE,
+ GameOnePlaylistIE,
+)
from .gamespot import GameSpotIE
+from .gamestar import GameStarIE
from .gametrailers import GametrailersIE
from .gdcvault import GDCVaultIE
from .generic import GenericIE
+from .godtube import GodTubeIE
from .googleplus import GooglePlusIE
from .googlesearch import GoogleSearchIE
from .gorillavid import GorillaVidIE
from .goshgay import GoshgayIE
+from .grooveshark import GroovesharkIE
from .hark import HarkIE
from .helsinki import HelsinkiIE
from .hentaistigma import HentaiStigmaIE
+from .hornbunny import HornBunnyIE
from .hotnewhiphop import HotNewHipHopIE
from .howcast import HowcastIE
+from .howstuffworks import HowStuffWorksIE
from .huffpost import HuffPostIE
from .hypem import HypemIE
from .iconosquare import IconosquareIE
IviIE,
IviCompilationIE
)
+from .izlesene import IzleseneIE
from .jadorecettepub import JadoreCettePubIE
from .jeuxvideo import JeuxVideoIE
+from .jove import JoveIE
from .jukebox import JukeboxIE
from .justintv import JustinTVIE
from .jpopsukitv import JpopsukiIE
from .kickstarter import KickStarterIE
from .keek import KeekIE
from .kontrtube import KontrTubeIE
+from .krasview import KrasViewIE
from .ku6 import Ku6IE
from .la7 import LA7IE
from .lifenews import LifeNewsIE
from .mdr import MDRIE
from .metacafe import MetacafeIE
from .metacritic import MetacriticIE
+from .ministrygrid import MinistryGridIE
from .mit import TechTVMITIE, MITIE, OCWMITIE
+from .mitele import MiTeleIE
from .mixcloud import MixcloudIE
+from .mlb import MLBIE
from .mpora import MporaIE
from .mofosex import MofosexIE
+from .mojvideo import MojvideoIE
from .mooshare import MooshareIE
from .morningstar import MorningstarIE
from .motherless import MotherlessIE
from .motorsport import MotorsportIE
+from .movieclips import MovieClipsIE
from .moviezine import MoviezineIE
from .movshare import MovShareIE
from .mtv import (
MTVIggyIE,
)
from .musicplayon import MusicPlayOnIE
+from .musicvault import MusicVaultIE
from .muzu import MuzuTVIE
from .myspace import MySpaceIE
from .myspass import MySpassIE
from .ninegag import NineGagIE
from .noco import NocoIE
from .normalboots import NormalbootsIE
+from .nosvideo import NosVideoIE
from .novamov import NovaMovIE
from .nowness import NownessIE
from .nowvideo import NowVideoIE
from .ntv import NTVIE
from .nytimes import NYTimesIE
from .nuvid import NuvidIE
-from .oe1 import OE1IE
from .ooyala import OoyalaIE
-from .orf import ORFIE
+from .orf import (
+ ORFTVthekIE,
+ ORFOE1IE,
+ ORFFM4IE,
+)
from .parliamentliveuk import ParliamentLiveUKIE
+from .patreon import PatreonIE
from .pbs import PBSIE
from .photobucket import PhotobucketIE
+from .playfm import PlayFMIE
from .playvid import PlayvidIE
from .podomatic import PodomaticIE
from .pornhd import PornHdIE
from .pornhub import PornHubIE
from .pornotube import PornotubeIE
+from .promptfile import PromptFileIE
from .prosiebensat1 import ProSiebenSat1IE
from .pyvideo import PyvideoIE
from .radiofrance import RadioFranceIE
from .rottentomatoes import RottenTomatoesIE
from .roxwel import RoxwelIE
from .rtbf import RTBFIE
+from .rtlnl import RtlXlIE
from .rtlnow import RTLnowIE
from .rts import RTSIE
-from .rtve import RTVEALaCartaIE
+from .rtve import RTVEALaCartaIE, RTVELiveIE
from .ruhd import RUHDIE
from .rutube import (
RutubeIE,
RutubePersonIE,
)
from .rutv import RUTVIE
+from .sapo import SapoIE
from .savefrom import SaveFromIE
+from .sbs import SBSIE
from .scivee import SciVeeIE
from .screencast import ScreencastIE
from .servingsys import ServingSysIE
+from .shared import SharedIE
from .sina import SinaIE
from .slideshare import SlideshareIE
from .slutload import SlutloadIE
SmotriUserIE,
SmotriBroadcastIE,
)
+from .snotr import SnotrIE
+from .sockshare import SockshareIE
from .sohu import SohuIE
from .soundcloud import (
SoundcloudIE,
from .spiegel import SpiegelIE
from .spiegeltv import SpiegeltvIE
from .spike import SpikeIE
+from .sportdeutschland import SportDeutschlandIE
from .stanfordoc import StanfordOpenClassroomIE
from .steam import SteamIE
from .streamcloud import StreamcloudIE
from .streamcz import StreamCZIE
+from .sunporno import SunPornoIE
from .swrmediathek import SWRMediathekIE
from .syfy import SyfyIE
from .sztvhu import SztvHuIE
from .thisav import ThisAVIE
from .tinypic import TinyPicIE
from .tlc import TlcIE, TlcDeIE
+from .tnaflix import TNAFlixIE
from .toutv import TouTvIE
from .toypics import ToypicsUserIE, ToypicsIE
from .traileraddict import TrailerAddictIE
from .tutv import TutvIE
from .tvigle import TvigleIE
from .tvp import TvpIE
+from .tvplay import TVPlayIE
+from .ubu import UbuIE
from .udemy import (
UdemyIE,
UdemyCourseIE
from .videopremium import VideoPremiumIE
from .videott import VideoTtIE
from .videoweed import VideoWeedIE
+from .vidme import VidmeIE
from .vimeo import (
VimeoIE,
VimeoChannelIE,
from .viki import VikiIE
from .vk import VKIE
from .vodlocker import VodlockerIE
+from .vporn import VpornIE
from .vube import VubeIE
from .vuclip import VuClipIE
from .vulture import VultureIE
from .washingtonpost import WashingtonPostIE
from .wat import WatIE
+from .wayofthemaster import WayOfTheMasterIE
from .wdr import (
WDRIE,
WDRMobileIE,
from .worldstarhiphop import WorldStarHipHopIE
from .wrzuta import WrzutaIE
from .xbef import XBefIE
+from .xboxclips import XboxClipsIE
from .xhamster import XHamsterIE
from .xnxx import XNXXIE
from .xvideos import XVideosIE
YoutubeUserIE,
YoutubeWatchLaterIE,
)
+
from .zdf import ZDFIE
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+
+
+class ABCIE(InfoExtractor):
+ IE_NAME = 'abc.net.au'
+ _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://www.abc.net.au/news/2014-07-25/bringing-asylum-seekers-to-australia-would-give/5624716',
+ 'md5': 'dad6f8ad011a70d9ddf887ce6d5d0742',
+ 'info_dict': {
+ 'id': '5624716',
+ 'ext': 'mp4',
+ 'title': 'Bringing asylum seekers to Australia would give them right to asylum claims: professor',
+ 'description': 'md5:ba36fa5e27e5c9251fd929d339aea4af',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+
+ urls_info_json = self._search_regex(
+ r'inlineVideoData\.push\((.*?)\);', webpage, 'video urls',
+ flags=re.DOTALL)
+ urls_info = json.loads(urls_info_json.replace('\'', '"'))
+ formats = [{
+ 'url': url_info['url'],
+ 'width': int(url_info['width']),
+ 'height': int(url_info['height']),
+ 'tbr': int(url_info['bitrate']),
+ 'filesize': int(url_info['filesize']),
+ } for url_info in urls_info]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': self._og_search_title(webpage),
+ 'formats': formats,
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ }
class AcademicEarthCourseIE(InfoExtractor):
_VALID_URL = r'^https?://(?:www\.)?academicearth\.org/playlists/(?P<id>[^?#/]+)'
IE_NAME = 'AcademicEarth:Course'
+ _TEST = {
+ 'url': 'http://academicearth.org/playlists/laws-of-nature/',
+ 'info_dict': {
+ 'id': 'laws-of-nature',
+ 'title': 'Laws of Nature',
+ 'description': 'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.',
+ },
+ 'playlist_count': 4,
+ }
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class AdultSwimIE(InfoExtractor):
+ _VALID_URL = r'https?://video\.adultswim\.com/(?P<path>.+?)(?:\.html)?(?:\?.*)?(?:#.*)?$'
+ _TEST = {
+ 'url': 'http://video.adultswim.com/rick-and-morty/close-rick-counters-of-the-rick-kind.html?x=y#title',
+ 'playlist': [
+ {
+ 'md5': '4da359ec73b58df4575cd01a610ba5dc',
+ 'info_dict': {
+ 'id': '8a250ba1450996e901453d7f02ca02f5',
+ 'ext': 'flv',
+ 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 1',
+ 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
+ 'uploader': 'Rick and Morty',
+ 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
+ }
+ },
+ {
+ 'md5': 'ffbdf55af9331c509d95350bd0cc1819',
+ 'info_dict': {
+ 'id': '8a250ba1450996e901453d7f4bd102f6',
+ 'ext': 'flv',
+ 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 2',
+ 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
+ 'uploader': 'Rick and Morty',
+ 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
+ }
+ },
+ {
+ 'md5': 'b92409635540304280b4b6c36bd14a0a',
+ 'info_dict': {
+ 'id': '8a250ba1450996e901453d7fa73c02f7',
+ 'ext': 'flv',
+ 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 3',
+ 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
+ 'uploader': 'Rick and Morty',
+ 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
+ }
+ },
+ {
+ 'md5': 'e8818891d60e47b29cd89d7b0278156d',
+ 'info_dict': {
+ 'id': '8a250ba1450996e901453d7fc8ba02f8',
+ 'ext': 'flv',
+ 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 4',
+ 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
+ 'uploader': 'Rick and Morty',
+ 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
+ }
+ }
+ ]
+ }
+
+ _video_extensions = {
+ '3500': 'flv',
+ '640': 'mp4',
+ '150': 'mp4',
+ 'ipad': 'm3u8',
+ 'iphone': 'm3u8'
+ }
+ _video_dimensions = {
+ '3500': (1280, 720),
+ '640': (480, 270),
+ '150': (320, 180)
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_path = mobj.group('path')
+
+ webpage = self._download_webpage(url, video_path)
+ episode_id = self._html_search_regex(r'<link rel="video_src" href="http://i\.adultswim\.com/adultswim/adultswimtv/tools/swf/viralplayer.swf\?id=([0-9a-f]+?)"\s*/?\s*>', webpage, 'episode_id')
+ title = self._og_search_title(webpage)
+
+ index_url = 'http://asfix.adultswim.com/asfix-svc/episodeSearch/getEpisodesByIDs?networkName=AS&ids=%s' % episode_id
+ idoc = self._download_xml(index_url, title, 'Downloading episode index', 'Unable to download episode index')
+
+ episode_el = idoc.find('.//episode')
+ show_title = episode_el.attrib.get('collectionTitle')
+ episode_title = episode_el.attrib.get('title')
+ thumbnail = episode_el.attrib.get('thumbnailUrl')
+ description = episode_el.find('./description').text.strip()
+
+ entries = []
+ segment_els = episode_el.findall('./segments/segment')
+
+ for part_num, segment_el in enumerate(segment_els):
+ segment_id = segment_el.attrib.get('id')
+ segment_title = '%s %s part %d' % (show_title, episode_title, part_num + 1)
+ thumbnail = segment_el.attrib.get('thumbnailUrl')
+ duration = segment_el.attrib.get('duration')
+
+ segment_url = 'http://asfix.adultswim.com/asfix-svc/episodeservices/getCvpPlaylist?networkName=AS&id=%s' % segment_id
+ idoc = self._download_xml(segment_url, segment_title, 'Downloading segment information', 'Unable to download segment information')
+
+ formats = []
+ file_els = idoc.findall('.//files/file')
+
+ for file_el in file_els:
+ bitrate = file_el.attrib.get('bitrate')
+ type = file_el.attrib.get('type')
+ width, height = self._video_dimensions.get(bitrate, (None, None))
+ formats.append({
+ 'format_id': '%s-%s' % (bitrate, type),
+ 'url': file_el.text,
+ 'ext': self._video_extensions.get(bitrate, 'mp4'),
+ # The bitrate may not be a number (for example: 'iphone')
+ 'tbr': int(bitrate) if bitrate.isdigit() else None,
+ 'height': height,
+ 'width': width
+ })
+
+ self._sort_formats(formats)
+
+ entries.append({
+ 'id': segment_id,
+ 'title': segment_title,
+ 'formats': formats,
+ 'uploader': show_title,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'description': description
+ })
+
+ return {
+ '_type': 'playlist',
+ 'id': episode_id,
+ 'display_id': video_path,
+ 'entries': entries,
+ 'title': '%s %s' % (show_title, episode_title),
+ 'description': description,
+ 'thumbnail': thumbnail
+ }
'id': '19540403',
'ext': 'mp4',
'title': 'Planes 2 Bande-annonce VF',
- 'description': 'md5:c4b1f7bd682a91de6491ada267ec0f4d',
+ 'description': 'md5:eeaffe7c2d634525e21159b93acf3b1e',
'thumbnail': 're:http://.*\.jpg',
},
}, {
'id': '19544709',
'ext': 'mp4',
'title': 'Dragons 2 - Bande annonce finale VF',
- 'description': 'md5:e74a4dc750894bac300ece46c7036490',
+ 'description': 'md5:71742e3a74b0d692c7fce0dd2017a4ac',
'thumbnail': 're:http://.*\.jpg',
},
}]
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ int_or_none,
+)
+
+
+class AnySexIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?anysex\.com/(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://anysex.com/156592/',
+ 'md5': '023e9fbb7f7987f5529a394c34ad3d3d',
+ 'info_dict': {
+ 'id': '156592',
+ 'ext': 'mp4',
+ 'title': 'Busty and sexy blondie in her bikini strips for you',
+ 'description': 'md5:de9e418178e2931c10b62966474e1383',
+ 'categories': ['Erotic'],
+ 'duration': 270,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
+
+ title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+ description = self._html_search_regex(
+ r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
+ thumbnail = self._html_search_regex(
+ r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
+
+ categories = re.findall(
+ r'<a href="http://anysex\.com/categories/[^"]+" title="[^"]*">([^<]+)</a>', webpage)
+
+ duration = parse_duration(self._search_regex(
+ r'<b>Duration:</b> (\d+:\d+)', webpage, 'duration', fatal=False))
+ view_count = int_or_none(self._html_search_regex(
+ r'<b>Views:</b> (\d+)', webpage, 'view count', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'categories': categories,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'age_limit': 18,
+ }
(?:$|\?)
'''
- _TEST = {
+ _TESTS = [{
'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
'md5': '18ef68f48740e86ae94b98da815eec42',
'info_dict': {
'title': 'U.S. Official Warns Of \'Largest Ever\' IRS Phone Scam',
},
'add_ie': ['FiveMin'],
- }
+ }, {
+ 'url': 'http://on.aol.com/playlist/brace-yourself---todays-weirdest-news-152147?icid=OnHomepageC4_Omg_Img#_videoid=518184316',
+ 'info_dict': {
+ 'id': '152147',
+ 'title': 'Brace Yourself - Today\'s Weirdest News',
+ },
+ 'playlist_mincount': 10,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
#coding: utf-8
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
_VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)'
_TEST = {
- u'url': u'http://www.aparat.com/v/wP8On',
- u'file': u'wP8On.mp4',
- u'md5': u'6714e0af7e0d875c5a39c4dc4ab46ad1',
- u'info_dict': {
- u"title": u"تیم گلکسی 11 - زومیت",
+ 'url': 'http://www.aparat.com/v/wP8On',
+ 'md5': '6714e0af7e0d875c5a39c4dc4ab46ad1',
+ 'info_dict': {
+ 'id': 'wP8On',
+ 'ext': 'mp4',
+ 'title': 'تیم گلکسی 11 - زومیت',
},
- #u'skip': u'Extremely unreliable',
+ # 'skip': 'Extremely unreliable',
}
def _real_extract(self, url):
# Note: There is an easier-to-parse configuration at
# http://www.aparat.com/video/video/config/videohash/%video_id
# but the URL in there does not work
- embed_url = (u'http://www.aparat.com/video/video/embed/videohash/' +
- video_id + u'/vt/frame')
+ embed_url = ('http://www.aparat.com/video/video/embed/videohash/' +
+ video_id + '/vt/frame')
webpage = self._download_webpage(embed_url, video_id)
video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage)
from .common import InfoExtractor
from ..utils import (
compat_urlparse,
+ int_or_none,
)
formats.append({
'url': format_url,
'format': format['type'],
- 'width': format['width'],
- 'height': int(format['height']),
+ 'width': int_or_none(format['width']),
+ 'height': int_or_none(format['height']),
})
self._sort_formats(formats)
from ..utils import (
determine_ext,
ExtractorError,
+ qualities,
+ compat_urllib_parse_urlparse,
+ compat_urllib_parse,
+ int_or_none,
+ parse_duration,
+ unified_strdate,
)
-class ARDIE(InfoExtractor):
- _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[^/\?]+)(?:\?.*)?'
+class ARDMediathekIE(InfoExtractor):
+ IE_NAME = 'ARD:mediathek'
+ _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
- _TEST = {
- 'url': 'http://www.ardmediathek.de/das-erste/guenther-jauch/edward-snowden-im-interview-held-oder-verraeter?documentId=19288786',
- 'file': '19288786.mp4',
- 'md5': '515bf47ce209fb3f5a61b7aad364634c',
+ _TESTS = [{
+ 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
+ 'file': '22429276.mp4',
+ 'md5': '469751912f1de0816a9fc9df8336476c',
'info_dict': {
- 'title': 'Edward Snowden im Interview - Held oder Verräter?',
- 'description': 'Edward Snowden hat alles aufs Spiel gesetzt, um die weltweite \xdcberwachung durch die Geheimdienste zu enttarnen. Nun stellt sich der ehemalige NSA-Mitarbeiter erstmals weltweit in einem TV-Interview den Fragen eines NDR-Journalisten. Die Sendung vom Sonntagabend.',
- 'thumbnail': 'http://www.ardmediathek.de/ard/servlet/contentblob/19/28/87/90/19288790/bild/2250037',
+ 'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?',
+ 'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, Über die Spionage-Affäre diskutieren Clemens Binninger, Katrin Göring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014',
},
'skip': 'Blocked outside of Germany',
- }
+ }, {
+ 'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
+ 'info_dict': {
+ 'id': '22490580',
+ 'ext': 'mp4',
+ 'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
+ 'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
+ },
+ 'skip': 'Blocked outside of Germany',
+ }]
def _real_extract(self, url):
# determine video id from url
else:
video_id = m.group('video_id')
+ urlp = compat_urllib_parse_urlparse(url)
+ url = urlp._replace(path=compat_urllib_parse.quote(urlp.path.encode('utf-8'))).geturl()
+
webpage = self._download_webpage(url, video_id)
+ if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
+ raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
+
title = self._html_search_regex(
[r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
r'<meta name="dcterms.title" content="(.*?)"/>',
r'<h4 class="headline">(.*?)</h4>'],
webpage, 'title')
description = self._html_search_meta(
- 'dcterms.abstract', webpage, 'description')
- thumbnail = self._og_search_thumbnail(webpage)
-
-
- media_info = self._download_json(
- 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
- # The second element of the _mediaArray contains the standard http urls
- streams = media_info['_mediaArray'][1]['_mediaStreamArray']
- if not streams:
- if '"fsk"' in webpage:
- raise ExtractorError('This video is only available after 20:00')
-
- formats = []
-
- for s in streams:
- if type(s['_stream']) == list:
- for index, url in enumerate(s['_stream'][::-1]):
- quality = s['_quality'] + index
- formats.append({
- 'quality': quality,
- 'url': url,
- 'format_id': '%s-%s' % (determine_ext(url), quality)
+ 'dcterms.abstract', webpage, 'description', default=None)
+ if description is None:
+ description = self._html_search_meta(
+ 'description', webpage, 'meta description')
+
+ # Thumbnail is sometimes not present.
+ # It is in the mobile version, but that seems to use a different URL
+ # structure altogether.
+ thumbnail = self._og_search_thumbnail(webpage, default=None)
+
+ media_streams = re.findall(r'''(?x)
+ mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
+ "([^"]+)"''', webpage)
+
+ if media_streams:
+ QUALITIES = qualities(['lo', 'hi', 'hq'])
+ formats = []
+ for furl in set(media_streams):
+ if furl.endswith('.f4m'):
+ fid = 'f4m'
+ else:
+ fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
+ fid = fid_m.group(1) if fid_m else None
+ formats.append({
+ 'quality': QUALITIES(fid),
+ 'format_id': fid,
+ 'url': furl,
+ })
+ else: # request JSON file
+ media_info = self._download_json(
+ 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
+ # The second element of the _mediaArray contains the standard http urls
+ streams = media_info['_mediaArray'][1]['_mediaStreamArray']
+ if not streams:
+ if '"fsk"' in webpage:
+ raise ExtractorError('This video is only available after 20:00')
+
+ formats = []
+ for s in streams:
+ if type(s['_stream']) == list:
+ for index, url in enumerate(s['_stream'][::-1]):
+ quality = s['_quality'] + index
+ formats.append({
+ 'quality': quality,
+ 'url': url,
+ 'format_id': '%s-%s' % (determine_ext(url), quality)
})
- continue
+ continue
- format = {
- 'quality': s['_quality'],
- 'url': s['_stream'],
- }
+ format = {
+ 'quality': s['_quality'],
+ 'url': s['_stream'],
+ }
- format['format_id'] = '%s-%s' % (
- determine_ext(format['url']), format['quality'])
+ format['format_id'] = '%s-%s' % (
+ determine_ext(format['url']), format['quality'])
- formats.append(format)
+ formats.append(format)
self._sort_formats(formats)
'formats': formats,
'thumbnail': thumbnail,
}
+
+
+class ARDIE(InfoExtractor):
+ _VALID_URL = '(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
+ _TEST = {
+ 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
+ 'md5': 'd216c3a86493f9322545e045ddc3eb35',
+ 'info_dict': {
+ 'display_id': 'die-story-im-ersten-mission-unter-falscher-flagge',
+ 'id': '100',
+ 'ext': 'mp4',
+ 'duration': 2600,
+ 'title': 'Die Story im Ersten: Mission unter falscher Flagge',
+ 'upload_date': '20140804',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('display_id')
+
+ player_url = mobj.group('mainurl') + '~playerXml.xml'
+ doc = self._download_xml(player_url, display_id)
+ video_node = doc.find('./video')
+ upload_date = unified_strdate(video_node.find('./broadcastDate').text)
+ thumbnail = video_node.find('.//teaserImage//variant/url').text
+
+ formats = []
+ for a in video_node.findall('.//asset'):
+ f = {
+ 'format_id': a.attrib['type'],
+ 'width': int_or_none(a.find('./frameWidth').text),
+ 'height': int_or_none(a.find('./frameHeight').text),
+ 'vbr': int_or_none(a.find('./bitrateVideo').text),
+ 'abr': int_or_none(a.find('./bitrateAudio').text),
+ 'vcodec': a.find('./codecVideo').text,
+ 'tbr': int_or_none(a.find('./totalBitrate').text),
+ }
+ if a.find('./serverPrefix').text:
+ f['url'] = a.find('./serverPrefix').text
+ f['playpath'] = a.find('./fileName').text
+ else:
+ f['url'] = a.find('./fileName').text
+ formats.append(f)
+ self._sort_formats(formats)
+
+ return {
+ 'id': mobj.group('id'),
+ 'formats': formats,
+ 'display_id': display_id,
+ 'title': video_node.find('./title').text,
+ 'duration': parse_duration(video_node.find('./duration').text),
+ 'upload_date': upload_date,
+ 'thumbnail': thumbnail,
+ }
+
def _extract_from_webpage(self, webpage, video_id, lang):
json_url = self._html_search_regex(
- r'arte_vp_url="(.*?)"', webpage, 'json vp url')
+ [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'],
+ webpage, 'json vp url')
return self._extract_from_json_url(json_url, video_id, lang)
def _extract_from_json_url(self, json_url, video_id, lang):
regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l]
return any(re.match(r, f['versionCode']) for r in regexes)
# Some formats may not be in the same language as the url
+ # TODO: Might want not to drop videos that does not match requested language
+ # but to process those formats with lower precedence
formats = filter(_match_lang, all_formats)
- formats = list(formats) # in python3 filter returns an iterator
+ formats = list(formats) # in python3 filter returns an iterator
if not formats:
# Some videos are only available in the 'Originalversion'
# they aren't tagged as being in French or German
- if all(f['versionCode'] == 'VO' or f['versionCode'] == 'VA' for f in all_formats):
- formats = all_formats
- else:
- raise ExtractorError(u'The formats list is empty')
+ # Sometimes there are neither videos of requested lang code
+ # nor original version videos available
+ # For such cases we just take all_formats as is
+ formats = all_formats
+ if not formats:
+ raise ExtractorError('The formats list is empty')
if re.match(r'[A-Z]Q', formats[0]['quality']) is not None:
def sort_key(f):
# It also uses the arte_vp_url url from the webpage to extract the information
class ArteTVCreativeIE(ArteTVPlus7IE):
IE_NAME = 'arte.tv:creative'
- _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/magazine?/(?P<id>.+)'
+ _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/(?:magazine?/)?(?P<id>[^?#]+)'
- _TEST = {
+ _TESTS = [{
'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
'info_dict': {
- 'id': '050489-002',
+ 'id': '72176',
'ext': 'mp4',
- 'title': 'Agentur Amateur / Agence Amateur #2 : Corporate Design',
+ 'title': 'Folge 2 - Corporate Design',
+ 'upload_date': '20131004',
},
- }
+ }, {
+ 'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion',
+ 'info_dict': {
+ 'id': '160676',
+ 'ext': 'mp4',
+ 'title': 'Monty Python live (mostly)',
+ 'description': 'Événement ! Quarante-cinq ans après leurs premiers succès, les légendaires Monty Python remontent sur scène.\n',
+ 'upload_date': '20140805',
+ }
+ }]
class ArteTVFutureIE(ArteTVPlus7IE):
_VALID_URL = r'https?://bambuser\.com/channel/(?P<user>.*?)(?:/|#|\?|$)'
# The maximum number we can get with each request
_STEP = 50
+ _TEST = {
+ 'url': 'http://bambuser.com/channel/pixelversity',
+ 'info_dict': {
+ 'title': 'pixelversity',
+ },
+ 'playlist_mincount': 60,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
req = compat_urllib_request.Request(req_url)
# Without setting this header, we wouldn't get any result
req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
- info_json = self._download_webpage(req, user,
- 'Downloading page %d' % i)
- results = json.loads(info_json)['result']
- if len(results) == 0:
+ data = self._download_json(
+ req, user, 'Downloading page %d' % i)
+ results = data['result']
+ if not results:
break
last_id = results[-1]['vid']
urls.extend(self.url_result(v['page'], 'Bambuser') for v in results)
IE_NAME = 'Bandcamp:album'
_VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))'
- _TEST = {
+ _TESTS = [{
'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
'playlist': [
{
'playlistend': 2
},
'skip': 'Bandcamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
- }
+ }, {
+ 'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
+ 'info_dict': {
+ 'title': 'Hierophany of the Open Grave',
+ },
+ 'playlist_mincount': 9,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class BeegIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://beeg.com/5416503',
+ 'md5': '634526ae978711f6b748fe0dd6c11f57',
+ 'info_dict': {
+ 'id': '5416503',
+ 'ext': 'mp4',
+ 'title': 'Sultry Striptease',
+ 'description': 'md5:6db3c6177972822aaba18652ff59c773',
+ 'categories': list, # NSFW
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ quality_arr = self._search_regex(
+ r'(?s)var\s+qualityArr\s*=\s*{\s*(.+?)\s*}', webpage, 'quality formats')
+
+ formats = [{
+ 'url': fmt[1],
+ 'format_id': fmt[0],
+ 'height': int(fmt[0][:-1]),
+ } for fmt in re.findall(r"'([^']+)'\s*:\s*'([^']+)'", quality_arr)]
+
+ self._sort_formats(formats)
+
+ title = self._html_search_regex(
+ r'<title>([^<]+)\s*-\s*beeg\.?</title>', webpage, 'title')
+
+ description = self._html_search_regex(
+ r'<meta name="description" content="([^"]*)"',
+ webpage, 'description', fatal=False)
+ thumbnail = self._html_search_regex(
+ r'\'previewer.url\'\s*:\s*"([^"]*)"',
+ webpage, 'thumbnail', fatal=False)
+
+ categories_str = self._html_search_regex(
+ r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
+ categories = (
+ None if categories_str is None
+ else categories_str.split(','))
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'categories': categories,
+ 'formats': formats,
+ 'age_limit': 18,
+ }
'height': int(m['h']),
})
elif m['type'] == 'original':
- duration = m['d']
+ duration = float(m['d'])
elif m['type'] == 'youtube':
yt_id = m['link']
self.to_screen('Youtube video detected: %s' % yt_id)
class BlipTVIE(SubtitlesInfoExtractor):
- _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+]+)))'
+ _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+_]+)))'
_TESTS = [
{
'uploader_id': '792887',
'duration': 279,
}
+ },
+ {
+ # https://bugzilla.redhat.com/show_bug.cgi?id=967465
+ 'url': 'http://a.blip.tv/api.swf#h6Uag5KbVwI',
+ 'md5': '314e87b1ebe7a48fcbfdd51b791ce5a6',
+ 'info_dict': {
+ 'id': '6573122',
+ 'ext': 'mov',
+ 'upload_date': '20130520',
+ 'description': 'Two hapless space marines argue over what to do when they realize they have an astronomically huge problem on their hands.',
+ 'title': 'Red vs. Blue Season 11 Trailer',
+ 'timestamp': 1369029609,
+ 'uploader': 'redvsblue',
+ 'uploader_id': '792887',
+ }
}
]
class BlipTVUserIE(InfoExtractor):
- _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
+ _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$'
_PAGE_SIZE = 12
IE_NAME = 'blip.tv:user'
_TEST = {
'url': 'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html',
- 'md5': '7bf08858ff7c203c870e8a6190e221e5',
+ # The md5 checksum changes
'info_dict': {
'id': 'qurhIVlJSB6hzkVi229d8g',
'ext': 'flv',
return {
'id': name.split('-')[-1],
'title': title,
- 'url': f4m_url,
- 'ext': 'flv',
+ 'formats': self._extract_f4m_formats(f4m_url, name),
'description': self._og_search_description(webpage),
'thumbnail': self._og_search_thumbnail(webpage),
}
from ..utils import (
ExtractorError,
int_or_none,
+ parse_duration,
)
class BRIE(InfoExtractor):
IE_DESC = 'Bayerischer Rundfunk Mediathek'
- _VALID_URL = r'https?://(?:www\.)?br\.de/(?:[a-z0-9\-]+/)+(?P<id>[a-z0-9\-]+)\.html'
+ _VALID_URL = r'https?://(?:www\.)?br\.de/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
_BASE_URL = 'http://www.br.de'
_TESTS = [
'info_dict': {
'id': '25e279aa-1ffd-40fd-9955-5325bd48a53a',
'ext': 'mp4',
- 'title': 'Am 1. und 2. August in Oberammergau',
- 'description': 'md5:dfd224e5aa6819bc1fcbb7826a932021',
+ 'title': 'Wenn das Traditions-Theater wackelt',
+ 'description': 'Heimatsound-Festival 2014: Wenn das Traditions-Theater wackelt',
+ 'duration': 34,
}
},
{
'ext': 'mp4',
'title': 'Über den Pass',
'description': 'Die Eroberung der Alpen: Über den Pass',
+ 'duration': 2588,
}
},
{
'ext': 'aac',
'title': '"Keine neuen Schulden im nächsten Jahr"',
'description': 'Haushaltsentwurf: "Keine neuen Schulden im nächsten Jahr"',
+ 'duration': 64,
}
},
{
'ext': 'mp4',
'title': 'Umweltbewusster Häuslebauer',
'description': 'Uwe Erdelt: Umweltbewusster Häuslebauer',
+ 'duration': 116,
}
},
{
'ext': 'mp4',
'title': 'Folge 1 - Metaphysik',
'description': 'Kant für Anfänger: Folge 1 - Metaphysik',
+ 'duration': 893,
'uploader': 'Eva Maria Steimle',
'upload_date': '20140117',
}
media = {
'id': xml_media.get('externalId'),
'title': xml_media.find('title').text,
+ 'duration': parse_duration(xml_media.find('duration').text),
'formats': self._extract_formats(xml_media.find('assets')),
'thumbnails': self._extract_thumbnails(xml_media.find('teaserImage/variants')),
'description': ' '.join(xml_media.find('shareTitle').text.splitlines()),
def _extract_brightcove_urls(cls, webpage):
"""Return a list of all Brightcove URLs from the webpage """
- url_m = re.search(r'<meta\s+property="og:video"\s+content="(http://c.brightcove.com/[^"]+)"', webpage)
+ url_m = re.search(
+ r'<meta\s+property="og:video"\s+content="(https?://(?:secure|c)\.brightcove.com/[^"]+)"',
+ webpage)
if url_m:
url = unescapeHTML(url_m.group(1))
# Some sites don't add it, we can't download with this url, for example:
# http://www.ktvu.com/videos/news/raw-video-caltrain-releases-video-of-man-almost/vCTZdY/
- if 'playerKey' in url:
+ if 'playerKey' in url or 'videoId' in url:
return [url]
matches = re.findall(
referer = smuggled_data.get('Referer', url)
return self._get_video_info(
videoPlayer[0], query_str, query, referer=referer)
- else:
+ elif 'playerKey' in query:
player_key = query['playerKey']
return self._get_playlist_info(player_key[0])
+ else:
+ raise ExtractorError(
+ 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
+ expected=True)
def _get_video_info(self, video_id, query_str, query, referer=None):
request_url = self._FEDERATED_URL_TEMPLATE % query_str
req.add_header('Referer', referer)
webpage = self._download_webpage(req, video_id)
+ error_msg = self._html_search_regex(
+ r"<h1>We're sorry.</h1>\s*<p>(.*?)</p>", webpage,
+ 'error message', default=None)
+ if error_msg is not None:
+ raise ExtractorError(
+ 'brightcove said: %s' % error_msg, expected=True)
+
self.report_extraction(video_id)
info = self._search_regex(r'var experienceJSON = ({.*});', webpage, 'json')
info = json.loads(info)['data']
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
class CBSIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/video/(?P<id>[^/]+)/.*'
+ _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)/(?P<id>[^/]+)/.*'
- _TEST = {
- u'url': u'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
- u'file': u'4JUVEwq3wUT7.flv',
- u'info_dict': {
- u'title': u'Connect Chat feat. Garth Brooks',
- u'description': u'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
- u'duration': 1495,
+ _TESTS = [{
+ 'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
+ 'info_dict': {
+ 'id': '4JUVEwq3wUT7',
+ 'ext': 'flv',
+ 'title': 'Connect Chat feat. Garth Brooks',
+ 'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
+ 'duration': 1495,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ '_skip': 'Blocked outside the US',
+ }, {
+ 'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/',
+ 'info_dict': {
+ 'id': 'WWF_5KqY3PK1',
+ 'ext': 'flv',
+ 'title': 'Live on Letterman - St. Vincent',
+ 'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.',
+ 'duration': 3221,
},
- u'params': {
+ 'params': {
# rtmp download
- u'skip_download': True,
+ 'skip_download': True,
},
- }
+ '_skip': 'Blocked outside the US',
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
webpage = self._download_webpage(url, video_id)
real_id = self._search_regex(
r"video\.settings\.pid\s*=\s*'([^']+)';",
- webpage, u'real video ID')
+ webpage, 'real video ID')
return self.url_result(u'theplatform:%s' % real_id)
'id': '85523671',
'ext': 'mp4',
'title': 'The Sunday Times - Icons',
- 'description': 'md5:3e1c0dc6047498d6728dcdaad0891762',
+ 'description': 'md5:a5f7ff82e2f7a9ed77473fe666954e84',
'uploader': 'Us',
'uploader_id': 'usfilms',
'upload_date': '20140131'
from __future__ import unicode_literals
+import json
import re
from .common import InfoExtractor
+from ..utils import int_or_none
-translation_table = {
+_translation_table = {
'a': 'h', 'd': 'e', 'e': 'v', 'f': 'o', 'g': 'f', 'i': 'd', 'l': 'n',
'm': 'a', 'n': 'm', 'p': 'u', 'q': 't', 'r': 's', 'v': 'p', 'x': 'r',
'y': 'l', 'z': 'i',
}
+def _decode(s):
+ return ''.join(_translation_table.get(c, c) for c in s)
+
+
class CliphunterIE(InfoExtractor):
IE_NAME = 'cliphunter'
'''
_TEST = {
'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo',
- 'file': '1012420.flv',
- 'md5': '15e7740f30428abf70f4223478dc1225',
+ 'md5': 'a2ba71eebf523859fe527a61018f723e',
'info_dict': {
+ 'id': '1012420',
+ 'ext': 'mp4',
'title': 'Fun Jynx Maze solo',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'age_limit': 18,
+ 'duration': 1317,
}
}
webpage = self._download_webpage(url, video_id)
+ video_title = self._search_regex(
+ r'mediaTitle = "([^"]+)"', webpage, 'title')
+
pl_fiji = self._search_regex(
r'pl_fiji = \'([^\']+)\'', webpage, 'video data')
pl_c_qual = self._search_regex(
r'pl_c_qual = "(.)"', webpage, 'video quality')
- video_title = self._search_regex(
- r'mediaTitle = "([^"]+)"', webpage, 'title')
-
- video_url = ''.join(translation_table.get(c, c) for c in pl_fiji)
-
+ video_url = _decode(pl_fiji)
formats = [{
'url': video_url,
- 'format_id': pl_c_qual,
+ 'format_id': 'default-%s' % pl_c_qual,
}]
+ qualities_json = self._search_regex(
+ r'var pl_qualities\s*=\s*(.*?);\n', webpage, 'quality info')
+ qualities_data = json.loads(qualities_json)
+
+ for i, t in enumerate(
+ re.findall(r"pl_fiji_([a-z0-9]+)\s*=\s*'([^']+')", webpage)):
+ quality_id, crypted_url = t
+ video_url = _decode(crypted_url)
+ f = {
+ 'format_id': quality_id,
+ 'url': video_url,
+ 'quality': i,
+ }
+ if quality_id in qualities_data:
+ qd = qualities_data[quality_id]
+ m = re.match(
+ r'''(?x)<b>(?P<width>[0-9]+)x(?P<height>[0-9]+)<\\/b>
+ \s*\(\s*(?P<tbr>[0-9]+)\s*kb\\/s''', qd)
+ if m:
+ f['width'] = int(m.group('width'))
+ f['height'] = int(m.group('height'))
+ f['tbr'] = int(m.group('tbr'))
+ formats.append(f)
+ self._sort_formats(formats)
+
+ thumbnail = self._search_regex(
+ r"var\s+mov_thumb\s*=\s*'([^']+)';",
+ webpage, 'thumbnail', fatal=False)
+ duration = int_or_none(self._search_regex(
+ r'pl_dur\s*=\s*([0-9]+)', webpage, 'duration', fatal=False))
+
return {
'id': video_id,
'title': video_title,
'formats': formats,
+ 'duration': duration,
+ 'age_limit': self._rta_search(webpage),
+ 'thumbnail': thumbnail,
}
raise ExtractorError('Cannot find video data')
video_id = vdata['id']
- title = vdata['headline']
+ title = vdata.get('headline')
+ if title is None:
+ title = vdata.get('title')
+ if title is None:
+ raise ExtractorError('Cannot find title!')
description = vdata.get('dek')
thumbnail = vdata.get('image', {}).get('path')
author = vdata.get('author')
class ComedyCentralIE(MTVServicesInfoExtractor):
- _VALID_URL = r'''(?x)https?://(?:www\.)?(comedycentral|cc)\.com/
- (video-clips|episodes|cc-studios|video-collections)
+ _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
+ (video-clips|episodes|cc-studios|video-collections|full-episodes)
/(?P<title>.*)'''
_FEED_URL = 'http://comedycentral.com/feeds/mrss/'
_TEST = {
- 'url': 'http://www.comedycentral.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother',
+ 'url': 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother',
'md5': 'c4f48e9eda1b16dd10add0744344b6d8',
'info_dict': {
'id': 'cef0cbb3-e776-4bc9-b62e-8016deccb354',
(?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/
((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|
(?P<clip>
- (?:(?:guests/[^/]+|videos|video-playlists|special-editions)/[^/]+/(?P<videotitle>[^/?#]+))
+ (?:(?:guests/[^/]+|videos|video-playlists|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+))
|(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
|(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
)|
(?P<interview>
extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?)))
(?:[?#].*|$)'''
- _TEST = {
+ _TESTS = [{
'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
'md5': '4e2f5cb088a83cd8cdb7756132f9739d',
'info_dict': {
'uploader': 'thedailyshow',
'title': 'thedailyshow kristen-stewart part 1',
}
- }
+ }, {
+ 'url': 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thecolbertreport.cc.com/videos/29w6fx/-realhumanpraise-for-fox-news',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thecolbertreport.cc.com/videos/gh6urb/neil-degrasse-tyson-pt--1?xrs=eml_col_031114',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thedailyshow.cc.com/guests/michael-lewis/3efna8/exclusive---michael-lewis-extended-interview-pt--3',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thedailyshow.cc.com/episodes/sy7yv0/april-8--2014---denis-leary',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thecolbertreport.cc.com/episodes/8ase07/april-8--2014---jane-goodall',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://thedailyshow.cc.com/news-team/michael-che/7wnfel/we-need-to-talk-about-israel',
+ 'only_matching': True,
+ }]
_available_formats = ['3500', '2200', '1700', '1200', '750', '400']
'ext': self._video_extensions.get(format, 'mp4'),
'height': h,
'width': w,
+
+ 'format_note': 'HTTP 400 at the moment (patches welcome!)',
+ 'preference': -100,
})
formats.append({
'format_id': 'rtmp-%s' % format,
+from __future__ import unicode_literals
+
import base64
import hashlib
import json
clean_html,
compiled_regex_type,
ExtractorError,
+ int_or_none,
RegexNotFoundError,
sanitize_filename,
unescapeHTML,
* vcodec Name of the video codec in use
* container Name of the container format
* filesize The number of bytes, if known in advance
+ * filesize_approx An estimate for the number of bytes
* player_url SWF Player URL (used for rtmpdump).
* protocol The protocol that will be used for the actual
download, lower-case.
format, irrespective of the file format.
-1 for default (order by other properties),
-2 or smaller for less than default.
+ * http_referer HTTP Referer header value to set.
+ * http_method HTTP method to use for the download.
+ * http_headers A dictionary of additional HTTP headers
+ to add to the request.
+ * http_post_data Additional data to send with a POST
+ request.
url: Final video URL.
ext: Video filename extension.
format: The video format, defaults to ext (used for --get-format)
upload_date: Video upload date (YYYYMMDD).
If not explicitly set, calculated from timestamp.
uploader_id: Nickname or id of the video uploader.
- location: Physical location of the video.
+ location: Physical location where the video was filmed.
subtitles: The subtitle file contents as a dictionary in the format
{language: subtitles}.
duration: Length of the video in seconds, as an integer.
self.report_download_webpage(video_id)
elif note is not False:
if video_id is None:
- self.to_screen(u'%s' % (note,))
+ self.to_screen('%s' % (note,))
else:
- self.to_screen(u'%s: %s' % (video_id, note))
+ self.to_screen('%s: %s' % (video_id, note))
try:
return self._downloader.urlopen(url_or_request)
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
if errnote is False:
return False
if errnote is None:
- errnote = u'Unable to download webpage'
- errmsg = u'%s: %s' % (errnote, compat_str(err))
+ errnote = 'Unable to download webpage'
+ errmsg = '%s: %s' % (errnote, compat_str(err))
if fatal:
raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
else:
url = url_or_request.get_full_url()
except AttributeError:
url = url_or_request
- self.to_screen(u'Dumping request to ' + url)
+ self.to_screen('Dumping request to ' + url)
dump = base64.b64encode(webpage_bytes).decode('ascii')
self._downloader.to_screen(dump)
if self._downloader.params.get('write_pages', False):
url = url_or_request
basen = '%s_%s' % (video_id, url)
if len(basen) > 240:
- h = u'___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
+ h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
basen = basen[:240 - len(h)] + h
raw_filename = basen + '.dump'
filename = sanitize_filename(raw_filename, restricted=True)
- self.to_screen(u'Saving request to ' + filename)
+ self.to_screen('Saving request to ' + filename)
with open(filename, 'wb') as outf:
outf.write(webpage_bytes)
except LookupError:
content = webpage_bytes.decode('utf-8', 'replace')
- if (u'<title>Access to this site is blocked</title>' in content and
- u'Websense' in content[:512]):
- msg = u'Access to this webpage has been blocked by Websense filtering software in your network.'
+ if ('<title>Access to this site is blocked</title>' in content and
+ 'Websense' in content[:512]):
+ msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
blocked_iframe = self._html_search_regex(
r'<iframe src="([^"]+)"', content,
- u'Websense information URL', default=None)
+ 'Websense information URL', default=None)
if blocked_iframe:
- msg += u' Visit %s for more details' % blocked_iframe
+ msg += ' Visit %s for more details' % blocked_iframe
raise ExtractorError(msg, expected=True)
return (content, urlh)
return content
def _download_xml(self, url_or_request, video_id,
- note=u'Downloading XML', errnote=u'Unable to download XML',
+ note='Downloading XML', errnote='Unable to download XML',
transform_source=None, fatal=True):
"""Return the xml as an xml.etree.ElementTree.Element"""
xml_string = self._download_webpage(
return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
def _download_json(self, url_or_request, video_id,
- note=u'Downloading JSON metadata',
- errnote=u'Unable to download JSON metadata',
- transform_source=None):
- json_string = self._download_webpage(url_or_request, video_id, note, errnote)
+ note='Downloading JSON metadata',
+ errnote='Unable to download JSON metadata',
+ transform_source=None,
+ fatal=True):
+ json_string = self._download_webpage(
+ url_or_request, video_id, note, errnote, fatal=fatal)
+ if (not fatal) and json_string is False:
+ return None
if transform_source:
json_string = transform_source(json_string)
try:
raise ExtractorError('Failed to download JSON', cause=ve)
def report_warning(self, msg, video_id=None):
- idstr = u'' if video_id is None else u'%s: ' % video_id
+ idstr = '' if video_id is None else '%s: ' % video_id
self._downloader.report_warning(
- u'[%s] %s%s' % (self.IE_NAME, idstr, msg))
+ '[%s] %s%s' % (self.IE_NAME, idstr, msg))
def to_screen(self, msg):
"""Print msg to screen, prefixing it with '[ie_name]'"""
- self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg))
+ self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
def report_extraction(self, id_or_name):
"""Report information extraction."""
- self.to_screen(u'%s: Extracting information' % id_or_name)
+ self.to_screen('%s: Extracting information' % id_or_name)
def report_download_webpage(self, video_id):
"""Report webpage download."""
- self.to_screen(u'%s: Downloading webpage' % video_id)
+ self.to_screen('%s: Downloading webpage' % video_id)
def report_age_confirmation(self):
"""Report attempt to confirm age."""
- self.to_screen(u'Confirming age')
+ self.to_screen('Confirming age')
def report_login(self):
"""Report attempt to log in."""
- self.to_screen(u'Logging in')
+ self.to_screen('Logging in')
#Methods for following #608
@staticmethod
else:
for p in pattern:
mobj = re.search(p, string, flags)
- if mobj: break
+ if mobj:
+ break
if os.name != 'nt' and sys.stderr.isatty():
- _name = u'\033[0;34m%s\033[0m' % name
+ _name = '\033[0;34m%s\033[0m' % name
else:
_name = name
elif default is not _NO_DEFAULT:
return default
elif fatal:
- raise RegexNotFoundError(u'Unable to extract %s' % _name)
+ raise RegexNotFoundError('Unable to extract %s' % _name)
else:
- self._downloader.report_warning(u'unable to extract %s; '
- u'please report this issue on http://yt-dl.org/bug' % _name)
+ self._downloader.report_warning('unable to extract %s; '
+ 'please report this issue on http://yt-dl.org/bug' % _name)
return None
def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
else:
raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
except (IOError, netrc.NetrcParseError) as err:
- self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
+ self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
return (username, password)
+ def _get_tfa_info(self):
+ """
+ Get the two-factor authentication info
+ TODO - asking the user will be required for sms/phone verify
+ currently just uses the command line option
+ If there's no info available, return None
+ """
+ if self._downloader is None:
+ return None
+ downloader_params = self._downloader.params
+
+ if downloader_params.get('twofactor', None) is not None:
+ return downloader_params['twofactor']
+
+ return None
+
# Helper functions for extracting OpenGraph info
@staticmethod
def _og_regexes(prop):
return unescapeHTML(escaped)
def _og_search_thumbnail(self, html, **kargs):
- return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)
+ return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
def _og_search_description(self, html, **kargs):
return self._og_search_property('description', html, fatal=False, **kargs)
return self._og_search_property('title', html, **kargs)
def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
- regexes = self._og_regexes('video')
- if secure: regexes = self._og_regexes('video:secure_url') + regexes
+ regexes = self._og_regexes('video') + self._og_regexes('video:url')
+ if secure:
+ regexes = self._og_regexes('video:secure_url') + regexes
return self._html_search_regex(regexes, html, name, **kargs)
def _og_search_url(self, html, **kargs):
display_name = name
return self._html_search_regex(
r'''(?ix)<meta
- (?=[^>]+(?:itemprop|name|property)=["\']%s["\'])
+ (?=[^>]+(?:itemprop|name|property)=["\']?%s["\']?)
[^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
html, display_name, fatal=fatal, **kwargs)
def _sort_formats(self, formats):
if not formats:
- raise ExtractorError(u'No video formats found')
+ raise ExtractorError('No video formats found')
def _formats_key(f):
# TODO remove the following workaround
if f.get('vcodec') == 'none': # audio only
if self._downloader.params.get('prefer_free_formats'):
- ORDER = [u'aac', u'mp3', u'm4a', u'webm', u'ogg', u'opus']
+ ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
else:
- ORDER = [u'webm', u'opus', u'ogg', u'mp3', u'aac', u'm4a']
+ ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
ext_preference = 0
try:
audio_ext_preference = ORDER.index(f['ext'])
audio_ext_preference = -1
else:
if self._downloader.params.get('prefer_free_formats'):
- ORDER = [u'flv', u'mp4', u'webm']
+ ORDER = ['flv', 'mp4', 'webm']
else:
- ORDER = [u'webm', u'flv', u'mp4']
+ ORDER = ['webm', 'flv', 'mp4']
try:
ext_preference = ORDER.index(f['ext'])
except ValueError:
f.get('abr') if f.get('abr') is not None else -1,
audio_ext_preference,
f.get('filesize') if f.get('filesize') is not None else -1,
+ f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
f.get('format_id'),
)
formats.sort(key=_formats_key)
def _sleep(self, timeout, video_id, msg_template=None):
if msg_template is None:
- msg_template = u'%(video_id)s: Waiting for %(timeout)s seconds'
+ msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
msg = msg_template % {'video_id': video_id, 'timeout': timeout}
self.to_screen(msg)
time.sleep(timeout)
+ def _extract_f4m_formats(self, manifest_url, video_id):
+ manifest = self._download_xml(
+ manifest_url, video_id, 'Downloading f4m manifest',
+ 'Unable to download f4m manifest')
+
+ formats = []
+ media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
+ for i, media_el in enumerate(media_nodes):
+ tbr = int_or_none(media_el.attrib.get('bitrate'))
+ format_id = 'f4m-%d' % (i if tbr is None else tbr)
+ formats.append({
+ 'format_id': format_id,
+ 'url': manifest_url,
+ 'ext': 'flv',
+ 'tbr': tbr,
+ 'width': int_or_none(media_el.attrib.get('width')),
+ 'height': int_or_none(media_el.attrib.get('height')),
+ })
+ self._sort_formats(formats)
+
+ return formats
+
+ def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None):
+ formats = [{
+ 'format_id': 'm3u8-meta',
+ 'url': m3u8_url,
+ 'ext': ext,
+ 'protocol': 'm3u8',
+ 'preference': -1,
+ 'resolution': 'multiple',
+ 'format_note': 'Quality selection URL',
+ }]
+
+ m3u8_doc = self._download_webpage(m3u8_url, video_id)
+ last_info = None
+ kv_rex = re.compile(
+ r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
+ for line in m3u8_doc.splitlines():
+ if line.startswith('#EXT-X-STREAM-INF:'):
+ last_info = {}
+ for m in kv_rex.finditer(line):
+ v = m.group('val')
+ if v.startswith('"'):
+ v = v[1:-1]
+ last_info[m.group('key')] = v
+ elif line.startswith('#') or not line.strip():
+ continue
+ else:
+ if last_info is None:
+ formats.append({'url': line})
+ continue
+ tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
+
+ f = {
+ 'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
+ 'url': line.strip(),
+ 'tbr': tbr,
+ 'ext': ext,
+ }
+ codecs = last_info.get('CODECS')
+ if codecs:
+ # TODO: looks like video codec is not always necessarily goes first
+ va_codecs = codecs.split(',')
+ if va_codecs[0]:
+ f['vcodec'] = va_codecs[0].partition('.')[0]
+ if len(va_codecs) > 1 and va_codecs[1]:
+ f['acodec'] = va_codecs[1].partition('.')[0]
+ resolution = last_info.get('RESOLUTION')
+ if resolution:
+ width_str, height_str = resolution.split('x')
+ f['width'] = int(width_str)
+ f['height'] = int(height_str)
+ formats.append(f)
+ last_info = {}
+ self._sort_formats(formats)
+ return formats
+
class SearchInfoExtractor(InfoExtractor):
"""
def _real_extract(self, query):
mobj = re.match(self._make_valid_url(), query)
if mobj is None:
- raise ExtractorError(u'Invalid search query "%s"' % query)
+ raise ExtractorError('Invalid search query "%s"' % query)
prefix = mobj.group('prefix')
query = mobj.group('query')
else:
n = int(prefix)
if n <= 0:
- raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query))
+ raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
elif n > self._MAX_RESULTS:
- self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
+ self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
n = self._MAX_RESULTS
return self._get_n_results(query, n)
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_iso8601,
+ str_to_int,
+)
+
+
+class CrackedIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?cracked\.com/video_(?P<id>\d+)_[\da-z-]+\.html'
+ _TEST = {
+ 'url': 'http://www.cracked.com/video_19006_4-plot-holes-you-didnt-notice-in-your-favorite-movies.html',
+ 'md5': '4b29a5eeec292cd5eca6388c7558db9e',
+ 'info_dict': {
+ 'id': '19006',
+ 'ext': 'mp4',
+ 'title': '4 Plot Holes You Didn\'t Notice in Your Favorite Movies',
+ 'description': 'md5:3b909e752661db86007d10e5ec2df769',
+ 'timestamp': 1405659600,
+ 'upload_date': '20140718',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(
+ [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'], webpage, 'video URL')
+
+ title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage)
+
+ timestamp = self._html_search_regex(r'<time datetime="([^"]+)"', webpage, 'upload date', fatal=False)
+ if timestamp:
+ timestamp = parse_iso8601(timestamp[:-6])
+
+ view_count = str_to_int(self._html_search_regex(
+ r'<span class="views" id="viewCounts">([\d,\.]+) Views</span>', webpage, 'view count', fatal=False))
+ comment_count = str_to_int(self._html_search_regex(
+ r'<span id="commentCounts">([\d,\.]+)</span>', webpage, 'comment count', fatal=False))
+
+ m = re.search(r'_(?P<width>\d+)X(?P<height>\d+)\.mp4$', video_url)
+ if m:
+ width = int(m.group('width'))
+ height = int(m.group('height'))
+ else:
+ width = height = None
+
+ return {
+ 'id': video_id,
+ 'url':video_url,
+ 'title': title,
+ 'description': description,
+ 'timestamp': timestamp,
+ 'view_count': view_count,
+ 'comment_count': comment_count,
+ 'height': height,
+ 'width': width,
+ }
\ No newline at end of file
import json
import base64
import zlib
+import xml.etree.ElementTree
from hashlib import sha1
from math import pow, sqrt, floor
intlist_to_bytes,
unified_strdate,
clean_html,
+ urlencode_postdata,
)
from ..aes import (
aes_cbc_decrypt,
'1080': ('80', '108'),
}
+ def _login(self):
+ (username, password) = self._get_login_info()
+ if username is None:
+ return
+ self.report_login()
+ login_url = 'https://www.crunchyroll.com/?a=formhandler'
+ data = urlencode_postdata({
+ 'formname': 'RpcApiUser_Login',
+ 'name': username,
+ 'password': password,
+ })
+ login_request = compat_urllib_request.Request(login_url, data)
+ login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ self._download_webpage(login_request, None, False, 'Wrong login info')
+
+
+ def _real_initialize(self):
+ self._login()
+
+
def _decrypt_subtitles(self, data, iv, id):
data = bytes_to_intlist(data)
iv = bytes_to_intlist(iv)
output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
return output
+ def _convert_subtitles_to_ass(self, subtitles):
+ output = ''
+
+ def ass_bool(strvalue):
+ assvalue = '0'
+ if strvalue == '1':
+ assvalue = '-1'
+ return assvalue
+
+ sub_root = xml.etree.ElementTree.fromstring(subtitles)
+ if not sub_root:
+ return output
+
+ output = '[Script Info]\n'
+ output += 'Title: %s\n' % sub_root.attrib["title"]
+ output += 'ScriptType: v4.00+\n'
+ output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"]
+ output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"]
+ output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"]
+ output += """ScaledBorderAndShadow: yes
+
+[V4+ Styles]
+Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+"""
+ for style in sub_root.findall('./styles/style'):
+ output += 'Style: ' + style.attrib["name"]
+ output += ',' + style.attrib["font_name"]
+ output += ',' + style.attrib["font_size"]
+ output += ',' + style.attrib["primary_colour"]
+ output += ',' + style.attrib["secondary_colour"]
+ output += ',' + style.attrib["outline_colour"]
+ output += ',' + style.attrib["back_colour"]
+ output += ',' + ass_bool(style.attrib["bold"])
+ output += ',' + ass_bool(style.attrib["italic"])
+ output += ',' + ass_bool(style.attrib["underline"])
+ output += ',' + ass_bool(style.attrib["strikeout"])
+ output += ',' + style.attrib["scale_x"]
+ output += ',' + style.attrib["scale_y"]
+ output += ',' + style.attrib["spacing"]
+ output += ',' + style.attrib["angle"]
+ output += ',' + style.attrib["border_style"]
+ output += ',' + style.attrib["outline"]
+ output += ',' + style.attrib["shadow"]
+ output += ',' + style.attrib["alignment"]
+ output += ',' + style.attrib["margin_l"]
+ output += ',' + style.attrib["margin_r"]
+ output += ',' + style.attrib["margin_v"]
+ output += ',' + style.attrib["encoding"]
+ output += '\n'
+
+ output += """
+[Events]
+Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+"""
+ for event in sub_root.findall('./events/event'):
+ output += 'Dialogue: 0'
+ output += ',' + event.attrib["start"]
+ output += ',' + event.attrib["end"]
+ output += ',' + event.attrib["style"]
+ output += ',' + event.attrib["name"]
+ output += ',' + event.attrib["margin_l"]
+ output += ',' + event.attrib["margin_r"]
+ output += ',' + event.attrib["margin_v"]
+ output += ',' + event.attrib["effect"]
+ output += ',' + event.attrib["text"]
+ output += '\n'
+
+ return output
+
def _real_extract(self,url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('video_id')
})
subtitles = {}
+ sub_format = self._downloader.params.get('subtitlesformat', 'srt')
for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\
video_id, note='Downloading subtitles for '+sub_name)
lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
if not lang_code:
continue
- subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
+ if sub_format == 'ass':
+ subtitles[lang_code] = self._convert_subtitles_to_ass(subtitle)
+ else:
+ subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
return {
'id': video_id,
'title': 'International Health Care Models',
'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
}
+ }, {
+ 'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
+ 'info_dict': {
+ 'id': '342759',
+ 'title': 'General Motors Ignition Switch Recall',
+ },
+ 'playlist_duration_sum': 14855,
}]
def _real_extract(self, url):
+#coding: utf-8
+from __future__ import unicode_literals
+
import re
import json
import itertools
"""Information Extractor for Dailymotion"""
_VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
- IE_NAME = u'dailymotion'
+ IE_NAME = 'dailymotion'
_FORMATS = [
- (u'stream_h264_ld_url', u'ld'),
- (u'stream_h264_url', u'standard'),
- (u'stream_h264_hq_url', u'hq'),
- (u'stream_h264_hd_url', u'hd'),
- (u'stream_h264_hd1080_url', u'hd180'),
+ ('stream_h264_ld_url', 'ld'),
+ ('stream_h264_url', 'standard'),
+ ('stream_h264_hq_url', 'hq'),
+ ('stream_h264_hd_url', 'hd'),
+ ('stream_h264_hd1080_url', 'hd180'),
]
_TESTS = [
{
- u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
- u'file': u'x33vw9.mp4',
- u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
- u'info_dict': {
- u"uploader": u"Amphora Alex and Van .",
- u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
+ 'url': 'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
+ 'md5': '392c4b85a60a90dc4792da41ce3144eb',
+ 'info_dict': {
+ 'id': 'x33vw9',
+ 'ext': 'mp4',
+ 'uploader': 'Amphora Alex and Van .',
+ 'title': 'Tutoriel de Youtubeur"DL DES VIDEO DE YOUTUBE"',
}
},
# Vevo video
{
- u'url': u'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
- u'file': u'USUV71301934.mp4',
- u'info_dict': {
- u'title': u'Roar (Official)',
- u'uploader': u'Katy Perry',
- u'upload_date': u'20130905',
+ 'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
+ 'info_dict': {
+ 'title': 'Roar (Official)',
+ 'id': 'USUV71301934',
+ 'ext': 'mp4',
+ 'uploader': 'Katy Perry',
+ 'upload_date': '20130905',
},
- u'params': {
- u'skip_download': True,
+ 'params': {
+ 'skip_download': True,
},
- u'skip': u'VEVO is only available in some countries',
+ 'skip': 'VEVO is only available in some countries',
},
# age-restricted video
{
- u'url': u'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
- u'file': u'xyh2zz.mp4',
- u'md5': u'0d667a7b9cebecc3c89ee93099c4159d',
- u'info_dict': {
- u'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
- u'uploader': 'HotWaves1012',
- u'age_limit': 18,
+ 'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
+ 'md5': '0d667a7b9cebecc3c89ee93099c4159d',
+ 'info_dict': {
+ 'id': 'xyh2zz',
+ 'ext': 'mp4',
+ 'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
+ 'uploader': 'HotWaves1012',
+ 'age_limit': 18,
}
-
}
]
webpage)
if m_vevo is not None:
vevo_id = m_vevo.group('id')
- self.to_screen(u'Vevo video detected: %s' % vevo_id)
- return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo')
+ self.to_screen('Vevo video detected: %s' % vevo_id)
+ return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
age_limit = self._rta_search(webpage)
embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id
embed_page = self._download_webpage(embed_url, video_id,
- u'Downloading embed page')
+ 'Downloading embed page')
info = self._search_regex(r'var info = ({.*?}),$', embed_page,
'video info', flags=re.MULTILINE)
info = json.loads(info)
'height': height,
})
if not formats:
- raise ExtractorError(u'Unable to extract video URL')
+ raise ExtractorError('Unable to extract video URL')
# subtitles
video_subtitles = self.extract_subtitles(video_id, webpage)
return
view_count = self._search_regex(
- r'video_views_count[^>]+>\s+([\d\.,]+)', webpage, u'view count', fatal=False)
+ r'video_views_count[^>]+>\s+([\d\.,]+)', webpage, 'view count', fatal=False)
if view_count is not None:
view_count = str_to_int(view_count)
'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
video_id, note=False)
except ExtractorError as err:
- self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
+ self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
return {}
info = json.loads(sub_list)
if (info['total'] > 0):
sub_lang_list = dict((l['language'], l['url']) for l in info['list'])
return sub_lang_list
- self._downloader.report_warning(u'video doesn\'t have subtitles')
+ self._downloader.report_warning('video doesn\'t have subtitles')
return {}
class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
- IE_NAME = u'dailymotion:playlist'
+ IE_NAME = 'dailymotion:playlist'
_VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
_MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"'
_PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
+ _TESTS = [{
+ 'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
+ 'info_dict': {
+ 'title': 'SPORT',
+ },
+ 'playlist_mincount': 20,
+ }]
def _extract_entries(self, id):
video_ids = []
for pagenum in itertools.count(1):
request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
webpage = self._download_webpage(request,
- id, u'Downloading page %s' % pagenum)
+ id, 'Downloading page %s' % pagenum)
video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
class DailymotionUserIE(DailymotionPlaylistIE):
- IE_NAME = u'dailymotion:user'
+ IE_NAME = 'dailymotion:user'
_VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
_PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
+ _TESTS = [{
+ 'url': 'https://www.dailymotion.com/user/nqtv',
+ 'info_dict': {
+ 'id': 'nqtv',
+ 'title': 'Rémi Gaillard',
+ },
+ 'playlist_mincount': 100,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
webpage = self._download_webpage(url, user)
full_user = unescapeHTML(self._html_search_regex(
r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
- webpage, u'user', flags=re.DOTALL))
+ webpage, 'user'))
return {
'_type': 'playlist',
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ float_or_none,
+ int_or_none,
+ clean_html,
+)
+
+
+class DBTVIE(InfoExtractor):
+ _VALID_URL = r'http://dbtv\.no/(?P<id>[0-9]+)#(?P<display_id>.+)'
+ _TEST = {
+ 'url': 'http://dbtv.no/3649835190001#Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
+ 'md5': 'b89953ed25dacb6edb3ef6c6f430f8bc',
+ 'info_dict': {
+ 'id': '33100',
+ 'display_id': 'Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
+ 'ext': 'mp4',
+ 'title': 'Skulle teste ut fornøyelsespark, men kollegaen var bare opptatt av bikinikroppen',
+ 'description': 'md5:1504a54606c4dde3e4e61fc97aa857e0',
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'timestamp': 1404039863.438,
+ 'upload_date': '20140629',
+ 'duration': 69.544,
+ 'view_count': int,
+ 'categories': list,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ data = self._download_json(
+ 'http://api.dbtv.no/discovery/%s' % video_id, display_id)
+
+ video = data['playlist'][0]
+
+ formats = [{
+ 'url': f['URL'],
+ 'vcodec': f.get('container'),
+ 'width': int_or_none(f.get('width')),
+ 'height': int_or_none(f.get('height')),
+ 'vbr': float_or_none(f.get('rate'), 1000),
+ 'filesize': int_or_none(f.get('size')),
+ } for f in video['renditions'] if 'URL' in f]
+
+ if not formats:
+ for url_key, format_id in [('URL', 'mp4'), ('HLSURL', 'hls')]:
+ if url_key in video:
+ formats.append({
+ 'url': video[url_key],
+ 'format_id': format_id,
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video['id'],
+ 'display_id': display_id,
+ 'title': video['title'],
+ 'description': clean_html(video['desc']),
+ 'thumbnail': video.get('splash') or video.get('thumb'),
+ 'timestamp': float_or_none(video.get('publishedAt'), 1000),
+ 'duration': float_or_none(video.get('length'), 1000),
+ 'view_count': int_or_none(video.get('views')),
+ 'categories': video.get('tags'),
+ 'formats': formats,
+ }
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class DFBIE(InfoExtractor):
+ IE_NAME = 'tv.dfb.de'
+ _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://tv.dfb.de/video/highlights-des-empfangs-in-berlin/9070/',
+ # The md5 is different each time
+ 'info_dict': {
+ 'id': '9070',
+ 'ext': 'flv',
+ 'title': 'Highlights des Empfangs in Berlin',
+ 'upload_date': '20140716',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ player_info = self._download_xml(
+ 'http://tv.dfb.de/server/hd_video.php?play=%s' % video_id,
+ video_id)
+ video_info = player_info.find('video')
+
+ f4m_info = self._download_xml(self._proto_relative_url(video_info.find('url').text.strip()), video_id)
+ token_el = f4m_info.find('token')
+ manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
+
+ return {
+ 'id': video_id,
+ 'title': video_info.find('title').text,
+ 'url': manifest_url,
+ 'ext': 'flv',
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'upload_date': ''.join(video_info.find('time_date').text.split('.')[::-1]),
+ }
import re
from .common import InfoExtractor
+from ..utils import compat_urllib_parse_unquote
class DropboxIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?dropbox[.]com/s/(?P<id>[a-zA-Z0-9]{15})/(?P<title>[^?#]*)'
_TEST = {
- 'url': 'https://www.dropbox.com/s/0qr9sai2veej4f8/THE_DOCTOR_GAMES.mp4',
- 'md5': '8ae17c51172fb7f93bdd6a214cc8c896',
+ 'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
'info_dict': {
- 'id': '0qr9sai2veej4f8',
+ 'id': 'nelirfsxnmcfbfh',
'ext': 'mp4',
- 'title': 'THE_DOCTOR_GAMES'
+ 'title': 'youtube-dl test video \'ä"BaW_jenozKc'
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
- title = os.path.splitext(mobj.group('title'))[0]
- video_url = url + '?dl=1'
+ fn = compat_urllib_parse_unquote(mobj.group('title'))
+ title = os.path.splitext(fn)[0]
+ video_url = (
+ re.sub(r'[?&]dl=0', '', url) +
+ ('?' if '?' in url else '&') + 'dl=1')
return {
'id': video_id,
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import str_to_int
+
+
+class DrTuberIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?drtuber\.com/video/(?P<id>\d+)/(?P<display_id>[\w-]+)'
+ _TEST = {
+ 'url': 'http://www.drtuber.com/video/1740434/hot-perky-blonde-naked-golf',
+ 'md5': '93e680cf2536ad0dfb7e74d94a89facd',
+ 'info_dict': {
+ 'id': '1740434',
+ 'display_id': 'hot-perky-blonde-naked-golf',
+ 'ext': 'mp4',
+ 'title': 'Hot Perky Blonde Naked Golf',
+ 'like_count': int,
+ 'dislike_count': int,
+ 'comment_count': int,
+ 'categories': list, # NSFW
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ video_url = self._html_search_regex(
+ r'<source src="([^"]+)"', webpage, 'video URL')
+
+ title = self._html_search_regex(
+ r'<title>([^<]+)\s*-\s*Free', webpage, 'title')
+
+ thumbnail = self._html_search_regex(
+ r'poster="([^"]+)"',
+ webpage, 'thumbnail', fatal=False)
+
+ like_count = str_to_int(self._html_search_regex(
+ r'<span id="rate_likes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
+ webpage, 'like count', fatal=False))
+ dislike_count = str_to_int(self._html_search_regex(
+ r'<span id="rate_dislikes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
+ webpage, 'like count', fatal=False))
+ comment_count = str_to_int(self._html_search_regex(
+ r'<span class="comments_count">([\d,\.]+)</span>',
+ webpage, 'comment count', fatal=False))
+
+ cats_str = self._html_search_regex(
+ r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
+ categories = None if cats_str is None else cats_str.split(' ')
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'url': video_url,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'like_count': like_count,
+ 'dislike_count': dislike_count,
+ 'comment_count': comment_count,
+ 'categories': categories,
+ 'age_limit': self._rta_search(webpage),
+ }
--- /dev/null
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class DumpIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:www\.)?dump\.com/(?P<id>[a-zA-Z0-9]+)/'
+
+ _TEST = {
+ 'url': 'http://www.dump.com/oneus/',
+ 'md5': 'ad71704d1e67dfd9e81e3e8b42d69d99',
+ 'info_dict': {
+ 'id': 'oneus',
+ 'ext': 'flv',
+ 'title': "He's one of us.",
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ }
+
+ def _real_extract(self, url):
+ m = re.match(self._VALID_URL, url)
+ video_id = m.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ video_url = self._search_regex(
+ r's1.addVariable\("file",\s*"([^"]+)"', webpage, 'video URL')
+
+ thumb = self._og_search_thumbnail(webpage)
+ title = self._search_regex(r'<b>([^"]+)</b>', webpage, 'title')
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': video_url,
+ 'thumbnail': thumb,
+ }
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
-from ..utils import determine_ext
class EbaumsWorldIE(InfoExtractor):
_VALID_URL = r'https?://www\.ebaumsworld\.com/video/watch/(?P<id>\d+)'
_TEST = {
- u'url': u'http://www.ebaumsworld.com/video/watch/83367677/',
- u'file': u'83367677.mp4',
- u'info_dict': {
- u'title': u'A Giant Python Opens The Door',
- u'description': u'This is how nightmares start...',
- u'uploader': u'jihadpizza',
+ 'url': 'http://www.ebaumsworld.com/video/watch/83367677/',
+ 'info_dict': {
+ 'id': '83367677',
+ 'ext': 'mp4',
+ 'title': 'A Giant Python Opens The Door',
+ 'description': 'This is how nightmares start...',
+ 'uploader': 'jihadpizza',
},
}
'id': video_id,
'title': config.find('title').text,
'url': video_url,
- 'ext': determine_ext(video_url),
'description': config.find('description').text,
'thumbnail': config.find('image').text,
'uploader': config.find('username').text,
+# coding: utf-8
+from __future__ import unicode_literals
+
import json
import random
import re
from .common import InfoExtractor
from ..utils import (
- ExtractorError,
+ compat_str,
)
IE_NAME = '8tracks'
_VALID_URL = r'https?://8tracks\.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
_TEST = {
- u"name": u"EightTracks",
- u"url": u"http://8tracks.com/ytdl/youtube-dl-test-tracks-a",
- u"playlist": [
+ "name": "EightTracks",
+ "url": "http://8tracks.com/ytdl/youtube-dl-test-tracks-a",
+ "info_dict": {
+ 'id': '1336550',
+ 'display_id': 'youtube-dl-test-tracks-a',
+ "description": "test chars: \"'/\\ä↭",
+ "title": "youtube-dl test tracks \"'/\\ä↭<>",
+ },
+ "playlist": [
{
- u"file": u"11885610.m4a",
- u"md5": u"96ce57f24389fc8734ce47f4c1abcc55",
- u"info_dict": {
- u"title": u"youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "96ce57f24389fc8734ce47f4c1abcc55",
+ "info_dict": {
+ "id": "11885610",
+ "ext": "m4a",
+ "title": "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
},
{
- u"file": u"11885608.m4a",
- u"md5": u"4ab26f05c1f7291ea460a3920be8021f",
- u"info_dict": {
- u"title": u"youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "4ab26f05c1f7291ea460a3920be8021f",
+ "info_dict": {
+ "id": "11885608",
+ "ext": "m4a",
+ "title": "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
},
{
- u"file": u"11885679.m4a",
- u"md5": u"d30b5b5f74217410f4689605c35d1fd7",
- u"info_dict": {
- u"title": u"youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "d30b5b5f74217410f4689605c35d1fd7",
+ "info_dict": {
+ "id": "11885679",
+ "ext": "m4a",
+ "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
},
{
- u"file": u"11885680.m4a",
- u"md5": u"4eb0a669317cd725f6bbd336a29f923a",
- u"info_dict": {
- u"title": u"youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "4eb0a669317cd725f6bbd336a29f923a",
+ "info_dict": {
+ "id": "11885680",
+ "ext": "m4a",
+ "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
},
{
- u"file": u"11885682.m4a",
- u"md5": u"1893e872e263a2705558d1d319ad19e8",
- u"info_dict": {
- u"title": u"PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "1893e872e263a2705558d1d319ad19e8",
+ "info_dict": {
+ "id": "11885682",
+ "ext": "m4a",
+ "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
},
{
- u"file": u"11885683.m4a",
- u"md5": u"b673c46f47a216ab1741ae8836af5899",
- u"info_dict": {
- u"title": u"PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "b673c46f47a216ab1741ae8836af5899",
+ "info_dict": {
+ "id": "11885683",
+ "ext": "m4a",
+ "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
},
{
- u"file": u"11885684.m4a",
- u"md5": u"1d74534e95df54986da7f5abf7d842b7",
- u"info_dict": {
- u"title": u"phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "1d74534e95df54986da7f5abf7d842b7",
+ "info_dict": {
+ "id": "11885684",
+ "ext": "m4a",
+ "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
},
{
- u"file": u"11885685.m4a",
- u"md5": u"f081f47af8f6ae782ed131d38b9cd1c0",
- u"info_dict": {
- u"title": u"phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
- u"uploader_id": u"ytdl"
+ "md5": "f081f47af8f6ae782ed131d38b9cd1c0",
+ "info_dict": {
+ "id": "11885685",
+ "ext": "m4a",
+ "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
+ "uploader_id": "ytdl"
}
}
]
}
-
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- if mobj is None:
- raise ExtractorError(u'Invalid URL: %s' % url)
playlist_id = mobj.group('id')
webpage = self._download_webpage(url, playlist_id)
- json_like = self._search_regex(r"PAGE.mix = (.*?);\n", webpage, u'trax information', flags=re.DOTALL)
+ json_like = self._search_regex(
+ r"(?s)PAGE.mix = (.*?);\n", webpage, 'trax information')
data = json.loads(json_like)
session = str(random.randint(0, 1000000000))
track_count = data['tracks_count']
first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
next_url = first_url
- res = []
+ entries = []
for i in range(track_count):
- api_json = self._download_webpage(next_url, playlist_id,
- note=u'Downloading song information %s/%s' % (str(i+1), track_count),
- errnote=u'Failed to download song information')
+ api_json = self._download_webpage(
+ next_url, playlist_id,
+ note='Downloading song information %d/%d' % (i + 1, track_count),
+ errnote='Failed to download song information')
api_data = json.loads(api_json)
- track_data = api_data[u'set']['track']
+ track_data = api_data['set']['track']
info = {
- 'id': track_data['id'],
+ 'id': compat_str(track_data['id']),
'url': track_data['track_file_stream_url'],
'title': track_data['performer'] + u' - ' + track_data['name'],
'raw_title': track_data['name'],
'uploader_id': data['user']['login'],
'ext': 'm4a',
}
- res.append(info)
- next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id'])
- return res
+ entries.append(info)
+ next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (
+ session, mix_id, track_data['id'])
+ return {
+ '_type': 'playlist',
+ 'entries': entries,
+ 'id': compat_str(mix_id),
+ 'display_id': playlist_id,
+ 'title': data.get('name'),
+ 'description': data.get('description'),
+ }
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ parse_iso8601,
+)
+
+
+class EllenTVIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?ellentv\.com/videos/(?P<id>[a-z0-9_-]+)'
+ _TEST = {
+ 'url': 'http://www.ellentv.com/videos/0-7jqrsr18/',
+ 'md5': 'e4af06f3bf0d5f471921a18db5764642',
+ 'info_dict': {
+ 'id': '0-7jqrsr18',
+ 'ext': 'mp4',
+ 'title': 'What\'s Wrong with These Photos? A Whole Lot',
+ 'timestamp': 1406876400,
+ 'upload_date': '20140801',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ timestamp = parse_iso8601(self._search_regex(
+ r'<span class="publish-date"><time datetime="([^"]+)">',
+ webpage, 'timestamp'))
+
+ return {
+ 'id': video_id,
+ 'title': self._og_search_title(webpage),
+ 'url': self._html_search_meta('VideoURL', webpage, 'url'),
+ 'timestamp': timestamp,
+ }
+
+
+class EllenTVClipsIE(InfoExtractor):
+ IE_NAME = 'EllenTV:clips'
+ _VALID_URL = r'https?://(?:www\.)?ellentv\.com/episodes/(?P<id>[a-z0-9_-]+)'
+ _TEST = {
+ 'url': 'http://www.ellentv.com/episodes/meryl-streep-vanessa-hudgens/',
+ 'info_dict': {
+ 'id': 'meryl-streep-vanessa-hudgens',
+ 'title': 'Meryl Streep, Vanessa Hudgens',
+ },
+ 'playlist_mincount': 9,
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ playlist_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, playlist_id)
+ playlist = self._extract_playlist(webpage)
+
+ return {
+ '_type': 'playlist',
+ 'id': playlist_id,
+ 'title': self._og_search_title(webpage),
+ 'entries': self._extract_entries(playlist)
+ }
+
+ def _extract_playlist(self, webpage):
+ json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json')
+ try:
+ return json.loads("[{" + json_string + "}]")
+ except ValueError as ve:
+ raise ExtractorError('Failed to download JSON', cause=ve)
+
+ def _extract_entries(self, playlist):
+ return [self.url_result(item['url'], 'EllenTV') for item in playlist]
from __future__ import unicode_literals
-import re
+from .tnaflix import TNAFlixIE
-from .common import InfoExtractor
+class EMPFlixIE(TNAFlixIE):
+ _VALID_URL = r'^https?://www\.empflix\.com/videos/(?P<display_id>[0-9a-zA-Z-]+)-(?P<id>[0-9]+)\.html'
+
+ _TITLE_REGEX = r'name="title" value="(?P<title>[^"]*)"'
+ _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
+ _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
-class EmpflixIE(InfoExtractor):
- _VALID_URL = r'^https?://www\.empflix\.com/videos/.*?-(?P<id>[0-9]+)\.html'
_TEST = {
'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
'md5': 'b1bc15b6412d33902d6e5952035fcabc',
'info_dict': {
'id': '33051',
+ 'display_id': 'Amateur-Finger-Fuck',
'ext': 'mp4',
'title': 'Amateur Finger Fuck',
'description': 'Amateur solo finger fucking.',
+ 'thumbnail': 're:https?://.*\.jpg$',
'age_limit': 18,
}
}
-
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
-
- webpage = self._download_webpage(url, video_id)
- age_limit = self._rta_search(webpage)
-
- video_title = self._html_search_regex(
- r'name="title" value="(?P<title>[^"]*)"', webpage, 'title')
- video_description = self._html_search_regex(
- r'name="description" value="([^"]*)"', webpage, 'description', fatal=False)
-
- cfg_url = self._html_search_regex(
- r'flashvars\.config = escape\("([^"]+)"',
- webpage, 'flashvars.config')
-
- cfg_xml = self._download_xml(
- cfg_url, video_id, note='Downloading metadata')
-
- formats = [
- {
- 'url': item.find('videoLink').text,
- 'format_id': item.find('res').text,
- } for item in cfg_xml.findall('./quality/item')
- ]
-
- return {
- 'id': video_id,
- 'title': video_title,
- 'description': video_description,
- 'formats': formats,
- 'age_limit': age_limit,
- }
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ str_to_int,
+)
+
+
+class EpornerIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\d+)/(?P<display_id>[\w-]+)'
+ _TEST = {
+ 'url': 'http://www.eporner.com/hd-porn/95008/Infamous-Tiffany-Teen-Strip-Tease-Video/',
+ 'md5': '3b427ae4b9d60619106de3185c2987cd',
+ 'info_dict': {
+ 'id': '95008',
+ 'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
+ 'ext': 'flv',
+ 'title': 'Infamous Tiffany Teen Strip Tease Video',
+ 'duration': 194,
+ 'view_count': int,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id)
+ title = self._html_search_regex(
+ r'<title>(.*?) - EPORNER', webpage, 'title')
+
+ redirect_code = self._html_search_regex(
+ r'<script type="text/javascript" src="/config5/%s/([a-f\d]+)/">' % video_id,
+ webpage, 'redirect_code')
+ redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, redirect_code)
+ player_code = self._download_webpage(
+ redirect_url, display_id, note='Downloading player config')
+
+ sources = self._search_regex(
+ r'(?s)sources\s*:\s*\[\s*({.+?})\s*\]', player_code, 'sources')
+
+ formats = []
+ for video_url, format_id in re.findall(r'file\s*:\s*"([^"]+)",\s*label\s*:\s*"([^"]+)"', sources):
+ fmt = {
+ 'url': video_url,
+ 'format_id': format_id,
+ }
+ m = re.search(r'^(\d+)', format_id)
+ if m:
+ fmt['height'] = int(m.group(1))
+ formats.append(fmt)
+ self._sort_formats(formats)
+
+ duration = parse_duration(self._search_regex(
+ r'class="mbtim">([0-9:]+)</div>', webpage, 'duration',
+ fatal=False))
+ view_count = str_to_int(self._search_regex(
+ r'id="cinemaviews">\s*([0-9,]+)\s*<small>views',
+ webpage, 'view count', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'formats': formats,
+ 'age_limit': self._rta_search(webpage),
+ }
r'<meta name="description" content="([^"]*)"',
webpage, 'description', fatal=False)
- playerUrl = self._og_search_video_url(webpage, name=u'player URL')
+ playerUrl = self._og_search_video_url(webpage, name='player URL')
title = self._html_search_regex(
r'<meta name="title" content="([^"]*)"',
class EveryonesMixtapeIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?everyonesmixtape\.com/#/mix/(?P<id>[0-9a-zA-Z]+)(?:/(?P<songnr>[0-9]))?$'
- _TEST = {
+ _TESTS = [{
'url': 'http://everyonesmixtape.com/#/mix/m7m0jJAbMQi/5',
- 'file': '5bfseWNmlds.mp4',
"info_dict": {
+ 'id': '5bfseWNmlds',
+ 'ext': 'mp4',
"title": "Passion Pit - \"Sleepyhead\" (Official Music Video)",
"uploader": "FKR.TV",
"uploader_id": "frenchkissrecords",
'params': {
'skip_download': True, # This is simply YouTube
}
- }
+ }, {
+ 'url': 'http://everyonesmixtape.com/#/mix/m7m0jJAbMQi',
+ 'info_dict': {
+ 'id': 'm7m0jJAbMQi',
+ 'title': 'Driving',
+ },
+ 'playlist_count': 24
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ unified_strdate,
+)
+
+
+class ExpoTVIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])'
+ _TEST = {
+ 'url': 'http://www.expotv.com/videos/reviews/1/24/LinneCardscom/17561',
+ 'md5': '2985e6d7a392b2f7a05e0ca350fe41d0',
+ 'info_dict': {
+ 'id': '17561',
+ 'ext': 'mp4',
+ 'upload_date': '20060212',
+ 'title': 'My Favorite Online Scrapbook Store',
+ 'view_count': int,
+ 'description': 'You\'ll find most everything you need at this virtual store front.',
+ 'uploader': 'Anna T.',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ player_key = self._search_regex(
+ r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
+ config_url = 'http://client.expotv.com/video/config/%s/%s' % (
+ video_id, player_key)
+ config = self._download_json(
+ config_url, video_id,
+ note='Downloading video configuration')
+
+ formats = [{
+ 'url': fcfg['file'],
+ 'height': int_or_none(fcfg.get('height')),
+ 'format_note': fcfg.get('label'),
+ 'ext': self._search_regex(
+ r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
+ 'file extension', default=None),
+ } for fcfg in config['sources']]
+ self._sort_formats(formats)
+
+ title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage)
+ thumbnail = config.get('image')
+ view_count = int_or_none(self._search_regex(
+ r'<h5>Plays: ([0-9]+)</h5>', webpage, 'view counts'))
+ uploader = self._search_regex(
+ r'<div class="reviewer">\s*<img alt="([^"]+)"', webpage, 'uploader',
+ fatal=False)
+ upload_date = unified_strdate(self._search_regex(
+ r'<h5>Reviewed on ([0-9/.]+)</h5>', webpage, 'upload date',
+ fatal=False))
+
+ return {
+ 'id': video_id,
+ 'formats': formats,
+ 'title': title,
+ 'description': description,
+ 'view_count': view_count,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ }
class FacebookIE(InfoExtractor):
_VALID_URL = r'''(?x)
https?://(?:\w+\.)?facebook\.com/
- (?:[^#?]*\#!/)?
- (?:video/video\.php|photo\.php|video/embed)\?(?:.*?)
+ (?:[^#]*?\#!/)?
+ (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
(?:v|video_id)=(?P<id>[0-9]+)
(?:.*)'''
_LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
_CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
_NETRC_MACHINE = 'facebook'
IE_NAME = 'facebook'
- _TEST = {
+ _TESTS = [{
'url': 'https://www.facebook.com/photo.php?v=120708114770723',
'md5': '48975a41ccc4b7a581abd68651c1a5a8',
'info_dict': {
'duration': 279,
'title': 'PEOPLE ARE AWESOME 2013',
}
- }
+ }, {
+ 'url': 'https://www.facebook.com/video.php?v=10204634152394104',
+ 'only_matching': True,
+ }]
def _login(self):
(useremail, password) = self._get_login_info()
check_form = {
'fb_dtsg': self._search_regex(r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg'),
- 'h': self._search_regex(r'name="h" value="(\w*?)"', login_results, 'h'),
+ 'h': self._search_regex(
+ r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
'name_action_selected': 'dont_save',
}
check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
ExtractorError,
compat_urllib_parse,
compat_urllib_request,
- determine_ext,
)
fields = dict(re.findall(r'''(?x)<input\s+
type="hidden"\s+
name="([^"]+)"\s+
- (?:id="[^"]+"\s+)?
value="([^"]*)"
''', webpage))
ext = self._search_regex(r'type:\s?\'([^\']+)\',',
webpage, 'extension', fatal=False)
video_url = self._search_regex(
- r'file:\s?\'(http[^\']+)\',', webpage, 'file url')
+ r'file:\s?loadURL\(\'(http[^\']+)\'\),', webpage, 'file url')
formats = [{
'format_id': 'sd',
+ video_id, video_id, 'Downloading XML config')
manifest_url = info.find('videos/video/url').text
- video_url = manifest_url.replace('manifest.f4m', 'index_2_av.m3u8')
- video_url = video_url.replace('/z/', '/i/')
+ manifest_url = manifest_url.replace('/z/', '/i/')
+
+ if manifest_url.startswith('rtmp'):
+ formats = [{'url': manifest_url, 'ext': 'flv'}]
+ else:
+ formats = []
+ available_formats = self._search_regex(r'/[^,]*,(.*?),k\.mp4', manifest_url, 'available formats')
+ for index, format_descr in enumerate(available_formats.split(',')):
+ format_info = {
+ 'url': manifest_url.replace('manifest.f4m', 'index_%d_av.m3u8' % index),
+ 'ext': 'mp4',
+ }
+ m_resolution = re.search(r'(?P<width>\d+)x(?P<height>\d+)', format_descr)
+ if m_resolution is not None:
+ format_info.update({
+ 'width': int(m_resolution.group('width')),
+ 'height': int(m_resolution.group('height')),
+ })
+ formats.append(format_info)
+
thumbnail_path = info.find('image').text
- return {'id': video_id,
- 'ext': 'flv' if video_url.startswith('rtmp') else 'mp4',
- 'url': video_url,
- 'title': info.find('titre').text,
- 'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', thumbnail_path),
- 'description': info.find('synopsis').text,
- }
+ return {
+ 'id': video_id,
+ 'title': info.find('titre').text,
+ 'formats': formats,
+ 'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', thumbnail_path),
+ 'description': info.find('synopsis').text,
+ }
class PluzzIE(FranceTVBaseInfoExtractor):
class FranceTvInfoIE(FranceTVBaseInfoExtractor):
IE_NAME = 'francetvinfo.fr'
- _VALID_URL = r'https?://www\.francetvinfo\.fr/.*/(?P<title>.+)\.html'
+ _VALID_URL = r'https?://(?:www|mobile)\.francetvinfo\.fr/.*/(?P<title>.+)\.html'
_TESTS = [{
'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
class CultureboxIE(FranceTVBaseInfoExtractor):
IE_NAME = 'culturebox.francetvinfo.fr'
- _VALID_URL = r'https?://culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)'
+ _VALID_URL = r'https?://(?:m\.)?culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)'
_TEST = {
'url': 'http://culturebox.francetvinfo.fr/einstein-on-the-beach-au-theatre-du-chatelet-146813',
'id': 'e402820827',
'ext': 'mp4',
'title': 'Please Use This Song (Jon Lajoie)',
- 'description': 'md5:2ed27d364f5a805a6dba199faaf6681d',
+ 'description': 'Please use this to sell something. www.jonlajoie.com',
'thumbnail': 're:^http:.*\.jpg$',
},
}]
'age_limit': age_limit,
'timestamp': timestamp,
}
+
+
+class GameOnePlaylistIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?gameone\.de(?:/tv)?/?$'
+ IE_NAME = 'gameone:playlist'
+ _TEST = {
+ 'url': 'http://www.gameone.de/tv',
+ 'info_dict': {
+ 'title': 'GameOne',
+ },
+ 'playlist_mincount': 294,
+ }
+
+ def _real_extract(self, url):
+ webpage = self._download_webpage('http://www.gameone.de/tv', 'TV')
+ max_id = max(map(int, re.findall(r'<a href="/tv/(\d+)"', webpage)))
+ entries = [
+ self.url_result('http://www.gameone.de/tv/%d' % video_id, 'GameOne')
+ for video_id in range(max_id, 0, -1)]
+
+ return {
+ '_type': 'playlist',
+ 'title': 'GameOne',
+ 'entries': entries,
+ }
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ parse_duration,
+ str_to_int,
+ unified_strdate,
+)
+
+
+class GameStarIE(InfoExtractor):
+ _VALID_URL = r'http://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
+ _TEST = {
+ 'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
+ 'md5': '96974ecbb7fd8d0d20fca5a00810cea7',
+ 'info_dict': {
+ 'id': '76110',
+ 'ext': 'mp4',
+ 'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
+ 'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den vollständigen Trailer an.',
+ 'thumbnail': 'http://images.gamestar.de/images/idgwpgsgp/bdb/2494525/600x.jpg',
+ 'upload_date': '20140728',
+ 'duration': 17
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ og_title = self._og_search_title(webpage)
+ title = og_title.replace(' - Video bei GameStar.de', '').strip()
+
+ url = 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id
+
+ description = self._og_search_description(webpage).strip()
+
+ thumbnail = self._proto_relative_url(
+ self._og_search_thumbnail(webpage), scheme='http:')
+
+ upload_date = unified_strdate(self._html_search_regex(
+ r'<span style="float:left;font-size:11px;">Datum: ([0-9]+\.[0-9]+\.[0-9]+) ',
+ webpage, 'upload_date', fatal=False))
+
+ duration = parse_duration(self._html_search_regex(
+ r' Länge: ([0-9]+:[0-9]+)</span>', webpage, 'duration',
+ fatal=False))
+
+ view_count = str_to_int(self._html_search_regex(
+ r' Zuschauer: ([0-9\.]+) ', webpage,
+ 'view_count', fatal=False))
+
+ comment_count = int_or_none(self._html_search_regex(
+ r'>Kommentieren \(([0-9]+)\)</a>', webpage, 'comment_count',
+ fatal=False))
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': url,
+ 'ext': 'mp4',
+ 'thumbnail': thumbnail,
+ 'description': description,
+ 'upload_date': upload_date,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'comment_count': comment_count
+ }
compat_urllib_request,
)
+
class GDCVaultIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)'
_TESTS = [
'skip_download': True, # Requires rtmpdump
}
},
+ {
+ 'url': 'http://www.gdcvault.com/play/1015301/Thexder-Meets-Windows-95-or',
+ 'md5': 'a5eb77996ef82118afbbe8e48731b98e',
+ 'info_dict': {
+ 'id': '1015301',
+ 'ext': 'flv',
+ 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
+ }
+ }
]
def _parse_mp4(self, xml_description):
webpage_url = 'http://www.gdcvault.com/play/' + video_id
start_page = self._download_webpage(webpage_url, video_id)
- xml_root = self._html_search_regex(r'<iframe src="(?P<xml_root>.*?)player.html.*?".*?</iframe>', start_page, 'xml root', None, False)
+ direct_url = self._search_regex(
+ r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);',
+ start_page, 'url', default=None)
+ if direct_url:
+ video_url = 'http://www.gdcvault.com/' + direct_url
+ title = self._html_search_regex(
+ r'<td><strong>Session Name</strong></td>\s*<td>(.*?)</td>',
+ start_page, 'title')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'flv',
+ 'title': title,
+ }
+ xml_root = self._html_search_regex(
+ r'<iframe src="(?P<xml_root>.*?)player.html.*?".*?</iframe>',
+ start_page, 'xml root', default=None)
if xml_root is None:
# Probably need to authenticate
- start_page = self._login(webpage_url, video_id)
- if start_page is None:
+ login_res = self._login(webpage_url, video_id)
+ if login_res is None:
self.report_warning('Could not login.')
else:
+ start_page = login_res
# Grab the url from the authenticated page
- xml_root = self._html_search_regex(r'<iframe src="(?P<xml_root>.*?)player.html.*?".*?</iframe>', start_page, 'xml root')
+ xml_root = self._html_search_regex(
+ r'<iframe src="(.*?)player.html.*?".*?</iframe>',
+ start_page, 'xml root')
- xml_name = self._html_search_regex(r'<iframe src=".*?\?xml=(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename', None, False)
+ xml_name = self._html_search_regex(
+ r'<iframe src=".*?\?xml=(.+?\.xml).*?".*?</iframe>',
+ start_page, 'xml filename', default=None)
if xml_name is None:
# Fallback to the older format
xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename')
from .common import InfoExtractor
from .youtube import YoutubeIE
from ..utils import (
- compat_urllib_error,
compat_urllib_parse,
- compat_urllib_request,
compat_urlparse,
compat_xml_parse_error,
+ determine_ext,
ExtractorError,
+ float_or_none,
HEADRequest,
+ orderedSet,
parse_xml,
smuggle_url,
unescapeHTML,
unified_strdate,
+ unsmuggle_url,
url_basename,
)
from .brightcove import BrightcoveIE
'description': 'Mario\'s life in the fast lane has never looked so good.',
},
},
+ # YouTube embed via <data-embed-url="">
+ {
+ 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
+ 'info_dict': {
+ 'id': 'jpSGZsgga_I',
+ 'ext': 'mp4',
+ 'title': 'Asphalt 8: Airborne - Launch Trailer',
+ 'uploader': 'Gameloft',
+ 'uploader_id': 'gameloft',
+ 'upload_date': '20130821',
+ 'description': 'md5:87bd95f13d8be3e7da87a5f2c443106a',
+ },
+ 'params': {
+ 'skip_download': True,
+ }
+ },
+ # Camtasia studio
+ {
+ 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
+ 'playlist': [{
+ 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
+ 'info_dict': {
+ 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
+ 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
+ 'ext': 'flv',
+ 'duration': 2235.90,
+ }
+ }, {
+ 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
+ 'info_dict': {
+ 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
+ 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
+ 'ext': 'flv',
+ 'duration': 2235.93,
+ }
+ }],
+ 'info_dict': {
+ 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
+ }
+ },
+ # Flowplayer
+ {
+ 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
+ 'md5': '9d65602bf31c6e20014319c7d07fba27',
+ 'info_dict': {
+ 'id': '5123ea6d5e5a7',
+ 'ext': 'mp4',
+ 'age_limit': 18,
+ 'uploader': 'www.handjobhub.com',
+ 'title': 'Busty Blonde Siri Tit Fuck While Wank at Handjob Hub',
+ }
+ },
+ # RSS feed
+ {
+ 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+ 'info_dict': {
+ 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+ 'title': 'Zero Punctuation',
+ 'description': 're:'
+ },
+ 'playlist_mincount': 11,
+ },
+ # Multiple brightcove videos
+ # https://github.com/rg3/youtube-dl/issues/2283
+ {
+ 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
+ 'info_dict': {
+ 'id': 'always-never',
+ 'title': 'Always / Never - The New Yorker',
+ },
+ 'playlist_count': 3,
+ 'params': {
+ 'extract_flat': False,
+ 'skip_download': True,
+ }
+ },
+ # MLB embed
+ {
+ 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
+ 'md5': '96f09a37e44da40dd083e12d9a683327',
+ 'info_dict': {
+ 'id': '33322633',
+ 'ext': 'mp4',
+ 'title': 'Ump changes call to ball',
+ 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
+ 'duration': 48,
+ 'timestamp': 1401537900,
+ 'upload_date': '20140531',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ },
]
def report_download_webpage(self, video_id):
"""Report information extraction."""
self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
- def _send_head(self, url):
- """Check if it is a redirect, like url shorteners, in case return the new url."""
-
- class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
- """
- Subclass the HTTPRedirectHandler to make it use our
- HEADRequest also on the redirected URL
- """
- def redirect_request(self, req, fp, code, msg, headers, newurl):
- if code in (301, 302, 303, 307):
- newurl = newurl.replace(' ', '%20')
- newheaders = dict((k,v) for k,v in req.headers.items()
- if k.lower() not in ("content-length", "content-type"))
- try:
- # This function was deprecated in python 3.3 and removed in 3.4
- origin_req_host = req.get_origin_req_host()
- except AttributeError:
- origin_req_host = req.origin_req_host
- return HEADRequest(newurl,
- headers=newheaders,
- origin_req_host=origin_req_host,
- unverifiable=True)
- else:
- raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
-
- class HTTPMethodFallback(compat_urllib_request.BaseHandler):
- """
- Fallback to GET if HEAD is not allowed (405 HTTP error)
- """
- def http_error_405(self, req, fp, code, msg, headers):
- fp.read()
- fp.close()
-
- newheaders = dict((k,v) for k,v in req.headers.items()
- if k.lower() not in ("content-length", "content-type"))
- return self.parent.open(compat_urllib_request.Request(req.get_full_url(),
- headers=newheaders,
- origin_req_host=req.get_origin_req_host(),
- unverifiable=True))
-
- # Build our opener
- opener = compat_urllib_request.OpenerDirector()
- for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
- HTTPMethodFallback, HEADRedirectHandler,
- compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
- opener.add_handler(handler())
-
- response = opener.open(HEADRequest(url))
- if response is None:
- raise ExtractorError('Invalid URL protocol')
- return response
-
def _extract_rss(self, url, video_id, doc):
playlist_title = doc.find('./channel/title').text
playlist_desc_el = doc.find('./channel/description')
'entries': entries,
}
+ def _extract_camtasia(self, url, video_id, webpage):
+ """ Returns None if no camtasia video can be found. """
+
+ camtasia_cfg = self._search_regex(
+ r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
+ webpage, 'camtasia configuration file', default=None)
+ if camtasia_cfg is None:
+ return None
+
+ title = self._html_search_meta('DC.title', webpage, fatal=True)
+
+ camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
+ camtasia_cfg = self._download_xml(
+ camtasia_url, video_id,
+ note='Downloading camtasia configuration',
+ errnote='Failed to download camtasia configuration')
+ fileset_node = camtasia_cfg.find('./playlist/array/fileset')
+
+ entries = []
+ for n in fileset_node.getchildren():
+ url_n = n.find('./uri')
+ if url_n is None:
+ continue
+
+ entries.append({
+ 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
+ 'title': '%s - %s' % (title, n.tag),
+ 'url': compat_urlparse.urljoin(url, url_n.text),
+ 'duration': float_or_none(n.find('./duration').text),
+ })
+
+ return {
+ '_type': 'playlist',
+ 'entries': entries,
+ 'title': title,
+ }
+
def _real_extract(self, url):
if url.startswith('//'):
return {
if not parsed_url.scheme:
default_search = self._downloader.params.get('default_search')
if default_search is None:
- default_search = 'error'
+ default_search = 'fixup_error'
- if default_search in ('auto', 'auto_warning'):
+ if default_search in ('auto', 'auto_warning', 'fixup_error'):
if '/' in url:
self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
return self.url_result('http://' + url)
- else:
+ elif default_search != 'fixup_error':
if default_search == 'auto_warning':
if re.match(r'^(?:url|URL)$', url):
raise ExtractorError(
self._downloader.report_warning(
'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
return self.url_result('ytsearch:' + url)
- elif default_search == 'error':
+
+ if default_search in ('error', 'fixup_error'):
raise ExtractorError(
('%r is not a valid URL. '
- 'Set --default-search "ytseach" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
+ 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
) % (url, url), expected=True)
else:
assert ':' in default_search
return self.url_result(default_search + url)
- video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
+
+ url, smuggled_data = unsmuggle_url(url)
+ force_videoid = None
+ if smuggled_data and 'force_videoid' in smuggled_data:
+ force_videoid = smuggled_data['force_videoid']
+ video_id = force_videoid
+ else:
+ video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
self.to_screen('%s: Requesting header' % video_id)
- try:
- response = self._send_head(url)
+ head_req = HEADRequest(url)
+ response = self._request_webpage(
+ head_req, video_id,
+ note=False, errnote='Could not send HEAD request to %s' % url,
+ fatal=False)
+ if response is not False:
# Check for redirect
new_url = response.geturl()
if url != new_url:
self.report_following_redirect(new_url)
+ if force_videoid:
+ new_url = smuggle_url(
+ new_url, {'force_videoid': force_videoid})
return self.url_result(new_url)
# Check for direct link to a video
'upload_date': upload_date,
}
- except compat_urllib_error.HTTPError:
- # This may be a stupid server that doesn't like HEAD, our UA, or so
- pass
-
try:
webpage = self._download_webpage(url, video_id)
except ValueError:
except compat_xml_parse_error:
pass
+ # Is it a Camtasia project?
+ camtasia_res = self._extract_camtasia(url, video_id, webpage)
+ if camtasia_res is not None:
+ return camtasia_res
+
# Sometimes embedded video player is hidden behind percent encoding
# (e.g. https://github.com/rg3/youtube-dl/issues/2448)
# Unescaping the whole page allows to handle those cases in a generic way
r'(?s)<title>(.*?)</title>', webpage, 'video title',
default='video')
+ # Try to detect age limit automatically
+ age_limit = self._rta_search(webpage)
+ # And then there are the jokers who advertise that they use RTA,
+ # but actually don't.
+ AGE_LIMIT_MARKERS = [
+ r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
+ ]
+ if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
+ age_limit = 18
+
# video uploader is domain name
video_uploader = self._search_regex(
r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
+ # Helper method
+ def _playlist_from_matches(matches, getter, ie=None):
+ urlrs = orderedSet(self.url_result(getter(m), ie) for m in matches)
+ return self.playlist_result(
+ urlrs, playlist_id=video_id, playlist_title=video_title)
+
# Look for BrightCove:
bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
if bc_urls:
matches = re.findall(r'''(?x)
(?:
<iframe[^>]+?src=|
+ data-video-url=|
<embed[^>]+?src=|
embedSWF\(?:\s*
)
(?:embed|v)/.+?)
\1''', webpage)
if matches:
- urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
- for tuppl in matches]
- return self.playlist_result(
- urlrs, playlist_id=video_id, playlist_title=video_title)
+ return _playlist_from_matches(
+ matches, lambda m: unescapeHTML(m[1]), ie='Youtube')
# Look for embedded Dailymotion player
matches = re.findall(
r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
if matches:
- urlrs = [self.url_result(unescapeHTML(tuppl[1]))
- for tuppl in matches]
- return self.playlist_result(
- urlrs, playlist_id=video_id, playlist_title=video_title)
+ return _playlist_from_matches(
+ matches, lambda m: unescapeHTML(m[1]))
# Look for embedded Wistia player
match = re.search(
mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
if mobj:
return self.url_result('http://blip.tv/a/a-'+mobj.group(1), 'BlipTV')
- mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9]+)', webpage)
+ mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
if mobj:
return self.url_result(mobj.group(1), 'BlipTV')
# Look for funnyordie embed
matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
if matches:
- urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie')
- for eurl in matches]
- return self.playlist_result(
- urlrs, playlist_id=video_id, playlist_title=video_title)
+ return _playlist_from_matches(
+ matches, getter=unescapeHTML, ie='FunnyOrDie')
# Look for embedded RUTV player
rutv_url = RUTVIE._extract_url(webpage)
url = unescapeHTML(mobj.group('url'))
return self.url_result(url, ie='MTVServicesEmbedded')
+ # Look for embedded yahoo player
+ mobj = re.search(
+ r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
+ webpage)
+ if mobj is not None:
+ return self.url_result(mobj.group('url'), 'Yahoo')
+
+ # Look for embedded sbs.com.au player
+ mobj = re.search(
+ r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1',
+ webpage)
+ if mobj is not None:
+ return self.url_result(mobj.group('url'), 'SBS')
+
+ mobj = re.search(
+ r'<iframe[^>]+?src=(["\'])(?P<url>https?://m\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
+ webpage)
+ if mobj is not None:
+ return self.url_result(mobj.group('url'), 'MLB')
+
# Start with something easy: JW Player in SWFObject
found = re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
if not found:
if not found:
# Broaden the findall a little bit: JWPlayer JS loader
found = re.findall(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage)
+ if not found:
+ # Flow player
+ found = re.findall(r'''(?xs)
+ flowplayer\("[^"]+",\s*
+ \{[^}]+?\}\s*,
+ \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
+ ["']?url["']?\s*:\s*["']([^"']+)["']
+ ''', webpage)
if not found:
# Try to find twitter cards info
found = re.findall(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
# We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
if m_video_type is not None:
- found = re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)
+ def check_video(vurl):
+ vpath = compat_urlparse.urlparse(vurl).path
+ vext = determine_ext(vpath)
+ return '.' in vpath and vext not in ('swf', 'png', 'jpg')
+ found = list(filter(
+ check_video,
+ re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)))
if not found:
# HTML5 video
- found = re.findall(r'(?s)<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage)
+ found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]+)? src="([^"]+)"', webpage)
if not found:
found = re.search(
r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
'url': video_url,
'uploader': video_uploader,
'title': video_title,
+ 'age_limit': age_limit,
})
if len(entries) == 1:
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ parse_iso8601,
+)
+
+
+class GodTubeIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?godtube\.com/watch/\?v=(?P<id>[\da-zA-Z]+)'
+ _TESTS = [
+ {
+ 'url': 'https://www.godtube.com/watch/?v=0C0CNNNU',
+ 'md5': '77108c1e4ab58f48031101a1a2119789',
+ 'info_dict': {
+ 'id': '0C0CNNNU',
+ 'ext': 'mp4',
+ 'title': 'Woman at the well.',
+ 'duration': 159,
+ 'timestamp': 1205712000,
+ 'uploader': 'beverlybmusic',
+ 'upload_date': '20080317',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ config = self._download_xml(
+ 'http://www.godtube.com/resource/mediaplayer/%s.xml' % video_id.lower(),
+ video_id, 'Downloading player config XML')
+
+ video_url = config.find('.//file').text
+ uploader = config.find('.//author').text
+ timestamp = parse_iso8601(config.find('.//date').text)
+ duration = parse_duration(config.find('.//duration').text)
+ thumbnail = config.find('.//image').text
+
+ media = self._download_xml(
+ 'http://www.godtube.com/media/xml/?v=%s' % video_id, video_id, 'Downloading media XML')
+
+ title = media.find('.//title').text
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'timestamp': timestamp,
+ 'uploader': uploader,
+ 'duration': duration,
+ }
_MAX_RESULTS = 1000
IE_NAME = 'video.google:search'
_SEARCH_KEY = 'gvsearch'
+ _TEST = {
+ 'url': 'gvsearch15:python language',
+ 'info_dict': {
+ 'id': 'python language',
+ 'title': 'python language',
+ },
+ 'playlist_count': 15,
+ }
def _get_n_results(self, query, n):
"""Get a specified number of results for a query"""
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import time
+import math
+import os.path
+import re
+
+
+from .common import InfoExtractor
+from ..utils import ExtractorError, compat_urllib_request, compat_html_parser
+
+from ..utils import (
+ compat_urllib_parse,
+ compat_urlparse,
+)
+
+
+class GroovesharkHtmlParser(compat_html_parser.HTMLParser):
+ def __init__(self):
+ self._current_object = None
+ self.objects = []
+ compat_html_parser.HTMLParser.__init__(self)
+
+ def handle_starttag(self, tag, attrs):
+ attrs = dict((k, v) for k, v in attrs)
+ if tag == 'object':
+ self._current_object = {'attrs': attrs, 'params': []}
+ elif tag == 'param':
+ self._current_object['params'].append(attrs)
+
+ def handle_endtag(self, tag):
+ if tag == 'object':
+ self.objects.append(self._current_object)
+ self._current_object = None
+
+ @classmethod
+ def extract_object_tags(cls, html):
+ p = cls()
+ p.feed(html)
+ p.close()
+ return p.objects
+
+
+class GroovesharkIE(InfoExtractor):
+ _VALID_URL = r'https?://(www\.)?grooveshark\.com/#!/s/([^/]+)/([^/]+)'
+ _TEST = {
+ 'url': 'http://grooveshark.com/#!/s/Jolene+Tenth+Key+Remix+Ft+Will+Sessions/6SS1DW?src=5',
+ 'md5': '7ecf8aefa59d6b2098517e1baa530023',
+ 'info_dict': {
+ 'id': '6SS1DW',
+ 'title': 'Jolene (Tenth Key Remix ft. Will Sessions)',
+ 'ext': 'mp3',
+ 'duration': 227,
+ }
+ }
+
+ do_playerpage_request = True
+ do_bootstrap_request = True
+
+ def _parse_target(self, target):
+ uri = compat_urlparse.urlparse(target)
+ hash = uri.fragment[1:].split('?')[0]
+ token = os.path.basename(hash.rstrip('/'))
+ return (uri, hash, token)
+
+ def _build_bootstrap_url(self, target):
+ (uri, hash, token) = self._parse_target(target)
+ query = 'getCommunicationToken=1&hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts)
+ return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token)
+
+ def _build_meta_url(self, target):
+ (uri, hash, token) = self._parse_target(target)
+ query = 'hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts)
+ return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token)
+
+ def _build_stream_url(self, meta):
+ return compat_urlparse.urlunparse(('http', meta['streamKey']['ip'], '/stream.php', None, None, None))
+
+ def _build_swf_referer(self, target, obj):
+ (uri, _, _) = self._parse_target(target)
+ return compat_urlparse.urlunparse((uri.scheme, uri.netloc, obj['attrs']['data'], None, None, None))
+
+ def _transform_bootstrap(self, js):
+ return re.split('(?m)^\s*try\s*{', js)[0] \
+ .split(' = ', 1)[1].strip().rstrip(';')
+
+ def _transform_meta(self, js):
+ return js.split('\n')[0].split('=')[1].rstrip(';')
+
+ def _get_meta(self, target):
+ (meta_url, token) = self._build_meta_url(target)
+ self.to_screen('Metadata URL: %s' % meta_url)
+
+ headers = {'Referer': compat_urlparse.urldefrag(target)[0]}
+ req = compat_urllib_request.Request(meta_url, headers=headers)
+ res = self._download_json(req, token,
+ transform_source=self._transform_meta)
+
+ if 'getStreamKeyWithSong' not in res:
+ raise ExtractorError(
+ 'Metadata not found. URL may be malformed, or Grooveshark API may have changed.')
+
+ if res['getStreamKeyWithSong'] is None:
+ raise ExtractorError(
+ 'Metadata download failed, probably due to Grooveshark anti-abuse throttling. Wait at least an hour before retrying from this IP.',
+ expected=True)
+
+ return res['getStreamKeyWithSong']
+
+ def _get_bootstrap(self, target):
+ (bootstrap_url, token) = self._build_bootstrap_url(target)
+
+ headers = {'Referer': compat_urlparse.urldefrag(target)[0]}
+ req = compat_urllib_request.Request(bootstrap_url, headers=headers)
+ res = self._download_json(req, token, fatal=False,
+ note='Downloading player bootstrap data',
+ errnote='Unable to download player bootstrap data',
+ transform_source=self._transform_bootstrap)
+ return res
+
+ def _get_playerpage(self, target):
+ (_, _, token) = self._parse_target(target)
+
+ webpage = self._download_webpage(
+ target, token,
+ note='Downloading player page',
+ errnote='Unable to download player page',
+ fatal=False)
+
+ if webpage is not None:
+ # Search (for example German) error message
+ error_msg = self._html_search_regex(
+ r'<div id="content">\s*<h2>(.*?)</h2>', webpage,
+ 'error message', default=None)
+ if error_msg is not None:
+ error_msg = error_msg.replace('\n', ' ')
+ raise ExtractorError('Grooveshark said: %s' % error_msg)
+
+ if webpage is not None:
+ o = GroovesharkHtmlParser.extract_object_tags(webpage)
+ return (webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
+
+ return (webpage, None)
+
+ def _real_initialize(self):
+ self.ts = int(time.time() * 1000) # timestamp in millis
+
+ def _real_extract(self, url):
+ (target_uri, _, token) = self._parse_target(url)
+
+ # 1. Fill cookiejar by making a request to the player page
+ swf_referer = None
+ if self.do_playerpage_request:
+ (_, player_objs) = self._get_playerpage(url)
+ if player_objs is not None:
+ swf_referer = self._build_swf_referer(url, player_objs[0])
+ self.to_screen('SWF Referer: %s' % swf_referer)
+
+ # 2. Ask preload.php for swf bootstrap data to better mimic webapp
+ if self.do_bootstrap_request:
+ bootstrap = self._get_bootstrap(url)
+ self.to_screen('CommunicationToken: %s' % bootstrap['getCommunicationToken'])
+
+ # 3. Ask preload.php for track metadata.
+ meta = self._get_meta(url)
+
+ # 4. Construct stream request for track.
+ stream_url = self._build_stream_url(meta)
+ duration = int(math.ceil(float(meta['streamKey']['uSecs']) / 1000000))
+ post_dict = {'streamKey': meta['streamKey']['streamKey']}
+ post_data = compat_urllib_parse.urlencode(post_dict).encode('utf-8')
+ headers = {
+ 'Content-Length': len(post_data),
+ 'Content-Type': 'application/x-www-form-urlencoded'
+ }
+ if swf_referer is not None:
+ headers['Referer'] = swf_referer
+
+ return {
+ 'id': token,
+ 'title': meta['song']['Name'],
+ 'http_method': 'POST',
+ 'url': stream_url,
+ 'ext': 'mp3',
+ 'format': 'mp3 audio',
+ 'duration': duration,
+ 'http_post_data': post_data,
+ 'http_headers': headers,
+ }
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ parse_duration,
+)
+
+
+class HornBunnyIE(InfoExtractor):
+ _VALID_URL = r'http?://(?:www\.)?hornbunny\.com/videos/(?P<title_dash>[a-z-]+)-(?P<id>\d+)\.html'
+ _TEST = {
+ 'url': 'http://hornbunny.com/videos/panty-slut-jerk-off-instruction-5227.html',
+ 'md5': '95e40865aedd08eff60272b704852ad7',
+ 'info_dict': {
+ 'id': '5227',
+ 'ext': 'flv',
+ 'title': 'panty slut jerk off instruction',
+ 'duration': 550,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(
+ url, video_id, note='Downloading initial webpage')
+ title = self._html_search_regex(
+ r'class="title">(.*?)</h2>', webpage, 'title')
+ redirect_url = self._html_search_regex(
+ r'pg&settings=(.*?)\|0"\);', webpage, 'title')
+ webpage2 = self._download_webpage(redirect_url, video_id)
+ video_url = self._html_search_regex(
+ r'flvMask:(.*?);', webpage2, 'video_url')
+
+ duration = parse_duration(self._search_regex(
+ r'<strong>Runtime:</strong>\s*([0-9:]+)</div>',
+ webpage, 'duration', fatal=False))
+ view_count = int_or_none(self._search_regex(
+ r'<strong>Views:</strong>\s*(\d+)</div>',
+ webpage, 'view count', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'ext': 'flv',
+ 'duration': duration,
+ 'view_count': view_count,
+ 'age_limit': 18,
+ }
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+import json
+import random
+import string
+
+from .common import InfoExtractor
+from ..utils import find_xpath_attr
+
+
+class HowStuffWorksIE(InfoExtractor):
+ _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
+ _TESTS = [
+ {
+ 'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
+ 'info_dict': {
+ 'id': '450221',
+ 'display_id': 'cool-jobs-iditarod-musher',
+ 'ext': 'flv',
+ 'title': 'Cool Jobs - Iditarod Musher',
+ 'description': 'md5:82bb58438a88027b8186a1fccb365f90',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ 'params': {
+ # md5 is not consistent
+ 'skip_download': True
+ }
+ },
+ {
+ 'url': 'http://adventure.howstuffworks.com/39516-deadliest-catch-jakes-farewell-pots-video.htm',
+ 'info_dict': {
+ 'id': '553470',
+ 'display_id': 'deadliest-catch-jakes-farewell-pots',
+ 'ext': 'mp4',
+ 'title': 'Deadliest Catch: Jake\'s Farewell Pots',
+ 'description': 'md5:9632c346d5e43ee238028c9cefd8dbbc',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ 'params': {
+ # md5 is not consistent
+ 'skip_download': True
+ }
+ },
+ {
+ 'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
+ 'info_dict': {
+ 'id': '440011',
+ 'display_id': 'sword-swallowing-1-by-dan-meyer',
+ 'ext': 'flv',
+ 'title': 'Sword Swallowing #1 by Dan Meyer',
+ 'description': 'md5:b2409e88172913e2e7d3d1159b0ef735',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ 'params': {
+ # md5 is not consistent
+ 'skip_download': True
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('id')
+ webpage = self._download_webpage(url, display_id)
+
+ content_id = self._search_regex(r'var siteSectionId="(\d+)";', webpage, 'content id')
+
+ mp4 = self._search_regex(
+ r'''(?xs)var\s+clip\s*=\s*{\s*
+ .+?\s*
+ content_id\s*:\s*%s\s*,\s*
+ .+?\s*
+ mp4\s*:\s*\[(.*?),?\]\s*
+ };\s*
+ videoData\.push\(clip\);''' % content_id,
+ webpage, 'mp4', fatal=False, default=None)
+
+ smil = self._download_xml(
+ 'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % content_id,
+ content_id, 'Downloading video SMIL')
+
+ http_base = find_xpath_attr(
+ smil,
+ './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
+ 'name',
+ 'httpBase').get('content')
+
+ def random_string(str_len=0):
+ return ''.join([random.choice(string.ascii_uppercase) for _ in range(str_len)])
+
+ URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=%s&g=%s' % (random_string(5), random_string(12))
+
+ formats = []
+
+ if mp4:
+ for video in json.loads('[%s]' % mp4):
+ bitrate = video['bitrate']
+ fmt = {
+ 'url': video['src'].replace('http://pmd.video.howstuffworks.com', http_base) + URL_SUFFIX,
+ 'format_id': bitrate,
+ }
+ m = re.search(r'(?P<vbr>\d+)[Kk]', bitrate)
+ if m:
+ fmt['vbr'] = int(m.group('vbr'))
+ formats.append(fmt)
+ else:
+ for video in smil.findall(
+ './/{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
+ vbr = int(video.attrib['system-bitrate']) / 1000
+ formats.append({
+ 'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
+ 'format_id': '%dk' % vbr,
+ 'vbr': vbr,
+ })
+
+ self._sort_formats(formats)
+
+ title = self._og_search_title(webpage)
+ TITLE_SUFFIX = ' : HowStuffWorks'
+ if title.endswith(TITLE_SUFFIX):
+ title = title[:-len(TITLE_SUFFIX)]
+
+ description = self._og_search_description(webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ return {
+ 'id': content_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
_DESCRIPTION_RE = [
r'<span class="page-object-description">(.+?)</span>',
r'id="my_show_video">.*?<p>(.*?)</p>',
+ r'<meta name="description" content="(.*?)"',
]
_TESTS = [
'skip_download': True,
},
},
+ {
+ 'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
+ 'md5': '4e9a0bda1e5eebd31ddcf86ec0b9b3c7',
+ 'info_dict': {
+ 'id': '078fdd005f6d3c02f63d795faa1b984f',
+ 'ext': 'mp4',
+ 'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
+ 'description': 'Giant skeletons, bloody hunts, and captivating'
+ ' natural beauty take our breath away.',
+ },
+ },
]
def _find_video_id(self, webpage):
r'data-video-id="(.+?)"',
r'<object id="vid_(.+?)"',
r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
+ r'class="hero-poster[^"]*?"[^>]*id="(.+?)"',
]
return self._search_regex(res_id, webpage, 'video id')
name_or_id = mobj.group('name_or_id')
page_type = mobj.group('type')
webpage = self._download_webpage(url, name_or_id)
- if page_type == 'articles':
- video_url = self._search_regex(r'var videoUrl = "(.+?)"', webpage, 'video url')
- return self.url_result(video_url, ie='IGN')
- elif page_type != 'video':
+ if page_type != 'video':
multiple_urls = re.findall(
'<param name="flashvars" value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
webpage)
IE_NAME = 'imdb:list'
IE_DESC = 'Internet Movie Database lists'
_VALID_URL = r'http://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
+ _TEST = {
+ 'url': 'http://www.imdb.com/list/JFs9NWw6XI0',
+ 'info_dict': {
+ 'id': 'JFs9NWw6XI0',
+ 'title': 'March 23, 2012 Releases',
+ },
+ 'playlist_count': 7,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
_VALID_URL = r'http://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
IE_DESC = 'Instagram user profile'
IE_NAME = 'instagram:user'
+ _TEST = {
+ 'url': 'http://instagram.com/porsche',
+ 'info_dict': {
+ 'id': 'porsche',
+ 'title': 'porsche',
+ },
+ 'playlist_mincount': 2,
+ 'playlist': [{
+ 'info_dict': {
+ 'id': '614605558512799803_462752227',
+ 'ext': 'mp4',
+ 'title': '#Porsche Intelligent Performance.',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'uploader': 'Porsche',
+ 'uploader_id': 'porsche',
+ 'timestamp': 1387486713,
+ 'upload_date': '20131219',
+ },
+ }],
+ 'params': {
+ 'extract_flat': True,
+ 'skip_download': True,
+ }
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
IE_DESC = 'ivi.ru compilations'
IE_NAME = 'ivi:compilation'
_VALID_URL = r'https?://(?:www\.)?ivi\.ru/watch/(?!\d+)(?P<compilationid>[a-z\d_-]+)(?:/season(?P<seasonid>\d+))?$'
+ _TESTS = [{
+ 'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa',
+ 'info_dict': {
+ 'id': 'dvoe_iz_lartsa',
+ 'title': 'Двое из ларца (2006 - 2008)',
+ },
+ 'playlist_mincount': 24,
+ }, {
+ 'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/season1',
+ 'info_dict': {
+ 'id': 'dvoe_iz_lartsa/season1',
+ 'title': 'Двое из ларца (2006 - 2008) 1 сезон',
+ },
+ 'playlist_mincount': 12,
+ }]
def _extract_entries(self, html, compilation_id):
return [self.url_result('http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), 'Ivi')
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ get_element_by_id,
+ parse_iso8601,
+ determine_ext,
+ int_or_none,
+ str_to_int,
+)
+
+
+class IzleseneIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:(?:www|m)\.)?izlesene\.com/(?:video|embedplayer)/(?:[^/]+/)?(?P<id>[0-9]+)'
+ _STREAM_URL = 'http://panel.izlesene.com/api/streamurl/{id:}/{format:}'
+ _TEST = {
+ 'url': 'http://www.izlesene.com/video/sevincten-cildirtan-dogum-gunu-hediyesi/7599694',
+ 'md5': '4384f9f0ea65086734b881085ee05ac2',
+ 'info_dict': {
+ 'id': '7599694',
+ 'ext': 'mp4',
+ 'title': 'Sevinçten Çıldırtan Doğum Günü Hediyesi',
+ 'description': 'Annesi oğluna doğum günü hediyesi olarak minecraft cd si alıyor, ve çocuk hunharca seviniyor',
+ 'thumbnail': 're:^http://.*\.jpg',
+ 'uploader_id': 'pelikzzle',
+ 'timestamp': 1404298698,
+ 'upload_date': '20140702',
+ 'duration': 95.395,
+ 'age_limit': 0,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ url = 'http://www.izlesene.com/video/%s' % video_id
+
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ uploader = self._html_search_regex(
+ r"adduserUsername\s*=\s*'([^']+)';", webpage, 'uploader', fatal=False, default='')
+ timestamp = parse_iso8601(self._html_search_meta(
+ 'uploadDate', webpage, 'upload date', fatal=False))
+
+ duration = int_or_none(self._html_search_regex(
+ r'"videoduration"\s*:\s*"([^"]+)"', webpage, 'duration', fatal=False))
+ if duration:
+ duration /= 1000.0
+
+ view_count = str_to_int(get_element_by_id('videoViewCount', webpage))
+ comment_count = self._html_search_regex(
+ r'comment_count\s*=\s*\'([^\']+)\';', webpage, 'uploader', fatal=False)
+
+ family_friendly = self._html_search_meta(
+ 'isFamilyFriendly', webpage, 'age limit', fatal=False)
+
+ content_url = self._html_search_meta(
+ 'contentURL', webpage, 'content URL', fatal=False)
+ ext = determine_ext(content_url, 'mp4')
+
+ # Might be empty for some videos.
+ qualities = self._html_search_regex(
+ r'"quality"\s*:\s*"([^"]+)"', webpage, 'qualities', fatal=False, default='')
+
+ formats = []
+ for quality in qualities.split('|'):
+ json = self._download_json(
+ self._STREAM_URL.format(id=video_id, format=quality), video_id,
+ note='Getting video URL for "%s" quality' % quality,
+ errnote='Failed to get video URL for "%s" quality' % quality
+ )
+ formats.append({
+ 'url': json.get('streamurl'),
+ 'ext': ext,
+ 'format_id': '%sp' % quality if quality else 'sd',
+ })
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'uploader_id': uploader,
+ 'timestamp': timestamp,
+ 'duration': duration,
+ 'view_count': int_or_none(view_count),
+ 'comment_count': int_or_none(comment_count),
+ 'age_limit': 18 if family_friendly == 'False' else 0,
+ 'formats': formats,
+ }
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ unified_strdate
+)
+
+
+class JoveIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P<id>[0-9]+)'
+ _CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}'
+ _TESTS = [
+ {
+ 'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current',
+ 'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b',
+ 'info_dict': {
+ 'id': '2744',
+ 'ext': 'mp4',
+ 'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation',
+ 'description': 'md5:015dd4509649c0908bc27f049e0262c6',
+ 'thumbnail': 're:^https?://.*\.png$',
+ 'upload_date': '20110523',
+ }
+ },
+ {
+ 'url': 'http://www.jove.com/video/51796/culturing-caenorhabditis-elegans-axenic-liquid-media-creation',
+ 'md5': '914aeb356f416811d911996434811beb',
+ 'info_dict': {
+ 'id': '51796',
+ 'ext': 'mp4',
+ 'title': 'Culturing Caenorhabditis elegans in Axenic Liquid Media and Creation of Transgenic Worms by Microparticle Bombardment',
+ 'description': 'md5:35ff029261900583970c4023b70f1dc9',
+ 'thumbnail': 're:^https?://.*\.png$',
+ 'upload_date': '20140802',
+ }
+ },
+
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ chapters_id = self._html_search_regex(
+ r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id')
+
+ chapters_xml = self._download_xml(
+ self._CHAPTERS_URL.format(video_id=chapters_id),
+ video_id, note='Downloading chapters XML',
+ errnote='Failed to download chapters XML')
+
+ video_url = chapters_xml.attrib.get('video')
+ if not video_url:
+ raise ExtractorError('Failed to get the video URL')
+
+ title = self._html_search_meta('citation_title', webpage, 'title')
+ thumbnail = self._og_search_thumbnail(webpage)
+ description = self._html_search_regex(
+ r'<div id="section_body_summary"><p class="jove_content">(.+?)</p>',
+ webpage, 'description', fatal=False)
+ publish_date = unified_strdate(self._html_search_meta(
+ 'citation_publication_date', webpage, 'publish date', fatal=False))
+ comment_count = self._html_search_regex(
+ r'<meta name="num_comments" content="(\d+) Comments?"',
+ webpage, 'comment count', fatal=False)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': video_url,
+ 'thumbnail': thumbnail,
+ 'description': description,
+ 'upload_date': publish_date,
+ 'comment_count': comment_count,
+ }
from __future__ import unicode_literals
+import itertools
import json
import os
import re
}
# Return count of items, list of *valid* items
- def _parse_page(self, url, video_id):
- info_json = self._download_webpage(url, video_id,
- 'Downloading video info JSON',
- 'unable to download video info JSON')
+ def _parse_page(self, url, video_id, counter):
+ info_json = self._download_webpage(
+ url, video_id,
+ 'Downloading video info JSON on page %d' % counter,
+ 'Unable to download video info JSON %d' % counter)
response = json.loads(info_json)
if type(response) != list:
entries = []
offset = 0
limit = self._JUSTIN_PAGE_LIMIT
- while True:
- if paged:
- self.report_download_page(video_id, offset)
+ for counter in itertools.count(1):
page_url = api + ('?offset=%d&limit=%d' % (offset, limit))
- page_count, page_info = self._parse_page(page_url, video_id)
+ page_count, page_info = self._parse_page(
+ page_url, video_id, counter)
entries.extend(page_info)
if not paged or page_count != limit:
break
_VALID_URL = r'^https?://(?:www\.)?khanacademy\.org/(?P<key>[^/]+)/(?:[^/]+/){,2}(?P<id>[^?#/]+)(?:$|[?#])'
IE_NAME = 'KhanAcademy'
- _TEST = {
+ _TESTS = [{
'url': 'http://www.khanacademy.org/video/one-time-pad',
- 'file': 'one-time-pad.mp4',
'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
'info_dict': {
+ 'id': 'one-time-pad',
+ 'ext': 'mp4',
'title': 'The one-time pad',
'description': 'The perfect cipher',
'duration': 176,
'uploader': 'Brit Cruise',
'upload_date': '20120411',
}
- }
+ }, {
+ 'url': 'https://www.khanacademy.org/math/applied-math/cryptography',
+ 'info_dict': {
+ 'id': 'cryptography',
+ 'title': 'Journey into cryptography',
+ 'description': 'How have humans protected their secret messages through history? What has changed today?',
+ },
+ 'playlist_mincount': 3,
+ }]
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
class KickStarterIE(InfoExtractor):
_VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>[^/]*)/.*'
- _TEST = {
+ _TESTS = [{
'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location',
'md5': 'c81addca81327ffa66c642b5d8b08cab',
'info_dict': {
'description': 'A unique motocross documentary that examines the '
'life and mind of one of sports most elite athletes: Josh Grant.',
},
- }
+ }, {
+ 'note': 'Embedded video (not using the native kickstarter video service)',
+ 'url': 'https://www.kickstarter.com/projects/597507018/pebble-e-paper-watch-for-iphone-and-android/posts/659178',
+ 'playlist': [
+ {
+ 'info_dict': {
+ 'id': '78704821',
+ 'ext': 'mp4',
+ 'uploader_id': 'pebble',
+ 'uploader': 'Pebble Technology',
+ 'title': 'Pebble iOS Notifications',
+ }
+ }
+ ],
+ }]
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
video_id = m.group('id')
webpage = self._download_webpage(url, video_id)
- video_url = self._search_regex(r'data-video-url="(.*?)"',
- webpage, 'video URL')
- video_title = self._html_search_regex(r'<title>(.*?)</title>',
- webpage, 'title').rpartition('— Kickstarter')[0].strip()
+ title = self._html_search_regex(
+ r'<title>\s*(.*?)(?:\s*— Kickstarter)?\s*</title>',
+ webpage, 'title')
+ video_url = self._search_regex(
+ r'data-video-url="(.*?)"',
+ webpage, 'video URL', default=None)
+ if video_url is None: # No native kickstarter, look for embedded videos
+ return {
+ '_type': 'url_transparent',
+ 'ie_key': 'Generic',
+ 'url': url,
+ 'title': title,
+ }
return {
'id': video_id,
'url': video_url,
- 'title': video_title,
+ 'title': title,
'description': self._og_search_description(webpage),
'thumbnail': self._og_search_thumbnail(webpage),
}
--- /dev/null
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ unescapeHTML,
+)
+
+
+class KrasViewIE(InfoExtractor):
+ IE_DESC = 'Красвью'
+ _VALID_URL = r'https?://krasview\.ru/video/(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://krasview.ru/video/512228',
+ 'md5': '3b91003cf85fc5db277870c8ebd98eae',
+ 'info_dict': {
+ 'id': '512228',
+ 'ext': 'mp4',
+ 'title': 'Снег, лёд, заносы',
+ 'description': 'Снято в городе Нягань, в Ханты-Мансийском автономном округе.',
+ 'duration': 27,
+ 'thumbnail': 're:^https?://.*\.jpg',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ flashvars = json.loads(self._search_regex(
+ r'flashvars\s*:\s*({.+?})\s*}\);', webpage, 'flashvars'))
+
+ video_url = flashvars['url']
+ title = unescapeHTML(flashvars['title'])
+ description = unescapeHTML(flashvars.get('subtitle') or self._og_search_description(webpage, default=None))
+ thumbnail = flashvars['image']
+ duration = int(flashvars['duration'])
+ filesize = int(flashvars['size'])
+ width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
+ height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'filesize': filesize,
+ 'width': width,
+ 'height': height,
+ }
from .common import InfoExtractor
from ..utils import (
+ compat_str,
compat_urllib_parse_urlparse,
compat_urlparse,
- xpath_with_ns,
- compat_str,
+ ExtractorError,
+ find_xpath_attr,
+ int_or_none,
orderedSet,
+ xpath_with_ns,
)
class LivestreamIE(InfoExtractor):
IE_NAME = 'livestream'
_VALID_URL = r'http://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>\d+))?/?$'
- _TEST = {
+ _TESTS = [{
'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
'md5': '53274c76ba7754fb0e8d072716f2292b',
'info_dict': {
'ext': 'mp4',
'title': 'Live from Webster Hall NYC',
'upload_date': '20121012',
+ 'like_count': int,
+ 'view_count': int,
+ 'thumbnail': 're:^http://.*\.jpg$'
}
- }
+ }, {
+ 'url': 'http://new.livestream.com/tedx/cityenglish',
+ 'info_dict': {
+ 'title': 'TEDCity2.0 (English)',
+ },
+ 'playlist_mincount': 4,
+ }]
+
+ def _parse_smil(self, video_id, smil_url):
+ formats = []
+ _SWITCH_XPATH = (
+ './/{http://www.w3.org/2001/SMIL20/Language}body/'
+ '{http://www.w3.org/2001/SMIL20/Language}switch')
+ smil_doc = self._download_xml(
+ smil_url, video_id,
+ note='Downloading SMIL information',
+ errnote='Unable to download SMIL information',
+ fatal=False)
+ if smil_doc is False: # Download failed
+ return formats
+ title_node = find_xpath_attr(
+ smil_doc, './/{http://www.w3.org/2001/SMIL20/Language}meta',
+ 'name', 'title')
+ if title_node is None:
+ self.report_warning('Cannot find SMIL id')
+ switch_node = smil_doc.find(_SWITCH_XPATH)
+ else:
+ title_id = title_node.attrib['content']
+ switch_node = find_xpath_attr(
+ smil_doc, _SWITCH_XPATH, 'id', title_id)
+ if switch_node is None:
+ raise ExtractorError('Cannot find switch node')
+ video_nodes = switch_node.findall(
+ '{http://www.w3.org/2001/SMIL20/Language}video')
+
+ for vn in video_nodes:
+ tbr = int_or_none(vn.attrib.get('system-bitrate'))
+ furl = (
+ 'http://livestream-f.akamaihd.net/%s?v=3.0.3&fp=WIN%%2014,0,0,145' %
+ (vn.attrib['src']))
+ if 'clipBegin' in vn.attrib:
+ furl += '&ssek=' + vn.attrib['clipBegin']
+ formats.append({
+ 'url': furl,
+ 'format_id': 'smil_%d' % tbr,
+ 'ext': 'flv',
+ 'tbr': tbr,
+ 'preference': -1000,
+ })
+ return formats
def _extract_video_info(self, video_data):
- video_url = video_data.get('progressive_url_hd') or video_data.get('progressive_url')
+ video_id = compat_str(video_data['id'])
+
+ FORMAT_KEYS = (
+ ('sd', 'progressive_url'),
+ ('hd', 'progressive_url_hd'),
+ )
+ formats = [{
+ 'format_id': format_id,
+ 'url': video_data[key],
+ 'quality': i + 1,
+ } for i, (format_id, key) in enumerate(FORMAT_KEYS)
+ if video_data.get(key)]
+
+ smil_url = video_data.get('smil_url')
+ if smil_url:
+ formats.extend(self._parse_smil(video_id, smil_url))
+ self._sort_formats(formats)
+
return {
- 'id': compat_str(video_data['id']),
- 'url': video_url,
- 'ext': 'mp4',
+ 'id': video_id,
+ 'formats': formats,
'title': video_data['caption'],
- 'thumbnail': video_data['thumbnail_url'],
+ 'thumbnail': video_data.get('thumbnail_url'),
'upload_date': video_data['updated_at'].replace('-', '')[:8],
+ 'like_count': video_data.get('likes', {}).get('total'),
+ 'view_count': video_data.get('views'),
}
def _real_extract(self, url):
event_name = mobj.group('event_name')
webpage = self._download_webpage(url, video_id or event_name)
+ og_video = self._og_search_video_url(
+ webpage, 'player url', fatal=False, default=None)
+ if og_video is not None:
+ query_str = compat_urllib_parse_urlparse(og_video).query
+ query = compat_urlparse.parse_qs(query_str)
+ if 'play_url' in query:
+ api_url = query['play_url'][0].replace('.smil', '')
+ info = json.loads(self._download_webpage(
+ api_url, video_id, 'Downloading video info'))
+ return self._extract_video_info(info)
+
+ config_json = self._search_regex(
+ r'window.config = ({.*?});', webpage, 'window config')
+ info = json.loads(config_json)['event']
+
+ def is_relevant(vdata, vid):
+ result = vdata['type'] == 'video'
+ if video_id is not None:
+ result = result and compat_str(vdata['data']['id']) == vid
+ return result
+
+ videos = [self._extract_video_info(video_data['data'])
+ for video_data in info['feed']['data']
+ if is_relevant(video_data, video_id)]
if video_id is None:
# This is an event page:
- config_json = self._search_regex(
- r'window.config = ({.*?});', webpage, 'window config')
- info = json.loads(config_json)['event']
- videos = [self._extract_video_info(video_data['data'])
- for video_data in info['feed']['data'] if video_data['type'] == 'video']
return self.playlist_result(videos, info['id'], info['full_name'])
else:
- og_video = self._og_search_video_url(webpage, 'player url')
- query_str = compat_urllib_parse_urlparse(og_video).query
- query = compat_urlparse.parse_qs(query_str)
- api_url = query['play_url'][0].replace('.smil', '')
- info = json.loads(self._download_webpage(
- api_url, video_id, 'Downloading video info'))
- return self._extract_video_info(info)
+ if not videos:
+ raise ExtractorError('Cannot find video %s' % video_id)
+ return videos[0]
# The original version of Livestream uses a different system
(?P<user>[^/]+)/(?P<type>video|folder)
(?:\?.*?Id=|/)(?P<id>.*?)(&|$)
'''
- _TEST = {
+ _TESTS = [{
'url': 'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
'info_dict': {
'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
# rtmp
'skip_download': True,
},
- }
+ }, {
+ 'url': 'https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
+ 'info_dict': {
+ 'id': 'a07bf706-d0e4-4e75-a747-b021d84f2fd3',
+ },
+ 'playlist_mincount': 4,
+ }]
def _extract_video(self, user, video_id):
api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
def _extract_folder(self, url, folder_id):
webpage = self._download_webpage(url, folder_id)
- urls = orderedSet(re.findall(r'<a href="(https?://livestre\.am/.*?)"', webpage))
+ paths = orderedSet(re.findall(
+ r'''(?x)(?:
+ <li\s+class="folder">\s*<a\s+href="|
+ <a\s+href="(?=https?://livestre\.am/)
+ )([^"]+)"''', webpage))
return {
'_type': 'playlist',
'id': folder_id,
'entries': [{
'_type': 'url',
- 'url': video_url,
- } for video_url in urls],
+ 'url': compat_urlparse.urljoin(url, p),
+ } for p in paths],
}
def _real_extract(self, url):
compat_urllib_request,
determine_ext,
ExtractorError,
+ int_or_none,
)
'skip_download': True,
},
},
+ # Movieclips.com video
+ {
+ 'url': 'http://www.metacafe.com/watch/mv-Wy7ZU/my_week_with_marilyn_do_you_love_me/',
+ 'info_dict': {
+ 'id': 'mv-Wy7ZU',
+ 'ext': 'mp4',
+ 'title': 'My Week with Marilyn - Do You Love Me?',
+ 'description': 'From the movie My Week with Marilyn - Colin (Eddie Redmayne) professes his love to Marilyn (Michelle Williams) and gets her to promise to return to set and finish the movie.',
+ 'uploader': 'movie_trailers',
+ 'duration': 176,
+ },
+ 'params': {
+ 'skip_download': 'requires rtmpdump',
+ }
+ }
]
def report_disclaimer(self):
# Extract URL, uploader and title from webpage
self.report_extraction(video_id)
+ video_url = None
mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
if mobj is not None:
mediaURL = compat_urllib_parse.unquote(mobj.group(1))
else:
gdaKey = mobj.group(1)
video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
- else:
+ if video_url is None:
mobj = re.search(r'<video src="([^"]+)"', webpage)
if mobj:
video_url = mobj.group(1)
video_ext = 'mp4'
- else:
- mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
- if mobj is None:
- raise ExtractorError('Unable to extract media URL')
- vardict = compat_parse_qs(mobj.group(1))
+ if video_url is None:
+ flashvars = self._search_regex(
+ r' name="flashvars" value="(.*?)"', webpage, 'flashvars',
+ default=None)
+ if flashvars:
+ vardict = compat_parse_qs(flashvars)
if 'mediaData' not in vardict:
raise ExtractorError('Unable to extract media URL')
mobj = re.search(
mediaURL = mobj.group('mediaURL').replace('\\/', '/')
video_url = '%s?__gda__=%s' % (mediaURL, mobj.group('key'))
video_ext = determine_ext(video_url)
-
- video_title = self._html_search_regex(r'(?im)<title>(.*) - Video</title>', webpage, 'title')
+ if video_url is None:
+ player_url = self._search_regex(
+ r"swfobject\.embedSWF\('([^']+)'",
+ webpage, 'config URL', default=None)
+ if player_url:
+ config_url = self._search_regex(
+ r'config=(.+)$', player_url, 'config URL')
+ config_doc = self._download_xml(
+ config_url, video_id,
+ note='Downloading video config')
+ smil_url = config_doc.find('.//properties').attrib['smil_file']
+ smil_doc = self._download_xml(
+ smil_url, video_id,
+ note='Downloading SMIL document')
+ base_url = smil_doc.find('./head/meta').attrib['base']
+ video_url = []
+ for vn in smil_doc.findall('.//video'):
+ br = int(vn.attrib['system-bitrate'])
+ play_path = vn.attrib['src']
+ video_url.append({
+ 'format_id': 'smil-%d' % br,
+ 'url': base_url,
+ 'play_path': play_path,
+ 'page_url': url,
+ 'player_url': player_url,
+ 'ext': play_path.partition(':')[0],
+ })
+
+ if video_url is None:
+ raise ExtractorError('Unsupported video type')
+
+ video_title = self._html_search_regex(
+ r'(?im)<title>(.*) - Video</title>', webpage, 'title')
description = self._og_search_description(webpage)
thumbnail = self._og_search_thumbnail(webpage)
video_uploader = self._html_search_regex(
r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
webpage, 'uploader nickname', fatal=False)
+ duration = int_or_none(
+ self._html_search_meta('video:duration', webpage))
+
+ age_limit = (
+ 18
+ if re.search(r'"contentRating":"restricted"', webpage)
+ else 0)
- if re.search(r'"contentRating":"restricted"', webpage) is not None:
- age_limit = 18
+ if isinstance(video_url, list):
+ formats = video_url
else:
- age_limit = 0
+ formats = [{
+ 'url': video_url,
+ 'ext': video_ext,
+ }]
+ self._sort_formats(formats)
return {
'id': video_id,
- 'url': video_url,
'description': description,
'uploader': video_uploader,
'title': video_title,
- 'thumbnail':thumbnail,
- 'ext': video_ext,
+ 'thumbnail': thumbnail,
'age_limit': age_limit,
+ 'formats': formats,
+ 'duration': duration,
}
--- /dev/null
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ smuggle_url,
+)
+
+
+class MinistryGridIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.ministrygrid.com/([^/?#]*/)*(?P<id>[^/#?]+)/?(?:$|[?#])'
+
+ _TEST = {
+ 'url': 'http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers',
+ 'md5': '844be0d2a1340422759c2a9101bab017',
+ 'info_dict': {
+ 'id': '3453494717001',
+ 'ext': 'mp4',
+ 'title': 'The Gospel by Numbers',
+ 'description': 'Coming soon from T4G 2014!',
+ 'uploader': 'LifeWay Christian Resources (MG)',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ portlets_json = self._search_regex(
+ r'Liferay\.Portlet\.list=(\[.+?\])', webpage, 'portlet list')
+ portlets = json.loads(portlets_json)
+ pl_id = self._search_regex(
+ r'<!--\s*p_l_id - ([0-9]+)<br>', webpage, 'p_l_id')
+
+ for i, portlet in enumerate(portlets):
+ portlet_url = 'http://www.ministrygrid.com/c/portal/render_portlet?p_l_id=%s&p_p_id=%s' % (pl_id, portlet)
+ portlet_code = self._download_webpage(
+ portlet_url, video_id,
+ note='Looking in portlet %s (%d/%d)' % (portlet, i + 1, len(portlets)),
+ fatal=False)
+ video_iframe_url = self._search_regex(
+ r'<iframe.*?src="([^"]+)"', portlet_code, 'video iframe',
+ default=None)
+ if video_iframe_url:
+ surl = smuggle_url(
+ video_iframe_url, {'force_videoid': video_id})
+ return {
+ '_type': 'url',
+ 'id': video_id,
+ 'url': surl,
+ }
+
+ raise ExtractorError('Could not find video iframe in any portlets')
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_parse,
+ get_element_by_attribute,
+ parse_duration,
+ strip_jsonp,
+)
+
+
+class MiTeleIE(InfoExtractor):
+ IE_NAME = 'mitele.es'
+ _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<episode>[^/]+)/'
+
+ _TEST = {
+ 'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
+ 'md5': '6a75fe9d0d3275bead0cb683c616fddb',
+ 'info_dict': {
+ 'id': '0fce117d',
+ 'ext': 'mp4',
+ 'title': 'Programa 144 - Tor, la web invisible',
+ 'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
+ 'display_id': 'programa-144',
+ 'duration': 2913,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ episode = mobj.group('episode')
+ webpage = self._download_webpage(url, episode)
+ embed_data_json = self._search_regex(
+ r'MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
+ flags=re.DOTALL
+ ).replace('\'', '"')
+ embed_data = json.loads(embed_data_json)
+
+ info_url = embed_data['flashvars']['host']
+ info_el = self._download_xml(info_url, episode).find('./video/info')
+
+ video_link = info_el.find('videoUrl/link').text
+ token_query = compat_urllib_parse.urlencode({'id': video_link})
+ token_info = self._download_json(
+ 'http://token.mitele.es/?' + token_query, episode,
+ transform_source=strip_jsonp
+ )
+
+ return {
+ 'id': embed_data['videoId'],
+ 'display_id': episode,
+ 'title': info_el.find('title').text,
+ 'url': token_info['tokenizedUrl'],
+ 'description': get_element_by_attribute('class', 'text', webpage),
+ 'thumbnail': info_el.find('thumb').text,
+ 'duration': parse_duration(info_el.find('duration').text),
+ }
from ..utils import (
compat_urllib_parse,
ExtractorError,
+ HEADRequest,
int_or_none,
parse_iso8601,
)
try:
# We only want to know if the request succeed
# don't download the whole file
- self._request_webpage(url, None, False)
+ self._request_webpage(HEADRequest(url), None, False)
return url
except ExtractorError:
url = None
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ parse_iso8601,
+ find_xpath_attr,
+)
+
+
+class MLBIE(InfoExtractor):
+ _VALID_URL = r'https?://m\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|shared/video/embed/embed\.html\?.*?\bcontent_id=)(?P<id>n?\d+)'
+ _TESTS = [
+ {
+ 'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
+ 'md5': 'ff56a598c2cf411a9a38a69709e97079',
+ 'info_dict': {
+ 'id': '34698933',
+ 'ext': 'mp4',
+ 'title': "Ackley's spectacular catch",
+ 'description': 'md5:7f5a981eb4f3cbc8daf2aeffa2215bf0',
+ 'duration': 66,
+ 'timestamp': 1405980600,
+ 'upload_date': '20140721',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ },
+ {
+ 'url': 'http://m.mlb.com/video/topic/81536970/v34496663/mianym-stanton-practices-for-the-home-run-derby',
+ 'md5': 'd9c022c10d21f849f49c05ae12a8a7e9',
+ 'info_dict': {
+ 'id': '34496663',
+ 'ext': 'mp4',
+ 'title': 'Stanton prepares for Derby',
+ 'description': 'md5:d00ce1e5fd9c9069e9c13ab4faedfa57',
+ 'duration': 46,
+ 'timestamp': 1405105800,
+ 'upload_date': '20140711',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ },
+ {
+ 'url': 'http://m.mlb.com/video/topic/vtp_hrd_sponsor/v34578115/hrd-cespedes-wins-2014-gillette-home-run-derby',
+ 'md5': '0e6e73d509321e142409b695eadd541f',
+ 'info_dict': {
+ 'id': '34578115',
+ 'ext': 'mp4',
+ 'title': 'Cespedes repeats as Derby champ',
+ 'description': 'md5:08df253ce265d4cf6fb09f581fafad07',
+ 'duration': 488,
+ 'timestamp': 1405399936,
+ 'upload_date': '20140715',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ },
+ {
+ 'url': 'http://m.mlb.com/video/v34577915/bautista-on-derby-captaining-duties-his-performance',
+ 'md5': 'b8fd237347b844365d74ea61d4245967',
+ 'info_dict': {
+ 'id': '34577915',
+ 'ext': 'mp4',
+ 'title': 'Bautista on Home Run Derby',
+ 'description': 'md5:b80b34031143d0986dddc64a8839f0fb',
+ 'duration': 52,
+ 'timestamp': 1405390722,
+ 'upload_date': '20140715',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ },
+ {
+ 'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
+ 'only_matching': True,
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ detail = self._download_xml(
+ 'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'
+ % (video_id[-3], video_id[-2], video_id[-1], video_id), video_id)
+
+ title = detail.find('./headline').text
+ description = detail.find('./big-blurb').text
+ duration = parse_duration(detail.find('./duration').text)
+ timestamp = parse_iso8601(detail.attrib['date'][:-5])
+
+ thumbnail = find_xpath_attr(
+ detail, './thumbnailScenarios/thumbnailScenario', 'type', '45').text
+
+ formats = []
+ for media_url in detail.findall('./url'):
+ playback_scenario = media_url.attrib['playback_scenario']
+ fmt = {
+ 'url': media_url.text,
+ 'format_id': playback_scenario,
+ }
+ m = re.search(r'(?P<vbr>\d+)K_(?P<width>\d+)X(?P<height>\d+)', playback_scenario)
+ if m:
+ fmt.update({
+ 'vbr': int(m.group('vbr')) * 1000,
+ 'width': int(m.group('width')),
+ 'height': int(m.group('height')),
+ })
+ formats.append(fmt)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ 'timestamp': timestamp,
+ 'formats': formats,
+ 'thumbnail': thumbnail,
+ }
+from __future__ import unicode_literals
+
import os
import re
compat_urllib_parse,
)
+
class MofosexIE(InfoExtractor):
- _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)'
+ _VALID_URL = r'^https?://(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)'
_TEST = {
- u'url': u'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
- u'file': u'5018.mp4',
- u'md5': u'1b2eb47ac33cc75d4a80e3026b613c5a',
- u'info_dict': {
- u"title": u"Japanese Teen Music Video",
- u"age_limit": 18,
+ 'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
+ 'md5': '1b2eb47ac33cc75d4a80e3026b613c5a',
+ 'info_dict': {
+ 'id': '5018',
+ 'ext': 'mp4',
+ 'title': 'Japanese Teen Music Video',
+ 'age_limit': 18,
}
}
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
- video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, u'title')
- video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, u'video_url'))
+ video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, 'title')
+ video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
path = compat_urllib_parse_urlparse(video_url).path
extension = os.path.splitext(path)[1][1:]
format = path.split('/')[5].split('_')[:2]
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ parse_duration,
+)
+
+
+class MojvideoIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?mojvideo\.com/video-(?P<display_id>[^/]+)/(?P<id>[a-f0-9]+)'
+ _TEST = {
+ 'url': 'http://www.mojvideo.com/video-v-avtu-pred-mano-rdecelaska-alfi-nipic/3d1ed4497707730b2906',
+ 'md5': 'f7fd662cc8ce2be107b0d4f2c0483ae7',
+ 'info_dict': {
+ 'id': '3d1ed4497707730b2906',
+ 'display_id': 'v-avtu-pred-mano-rdecelaska-alfi-nipic',
+ 'ext': 'mp4',
+ 'title': 'V avtu pred mano rdečelaska - Alfi Nipič',
+ 'thumbnail': 're:^http://.*\.jpg$',
+ 'duration': 242,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ # XML is malformed
+ playerapi = self._download_webpage(
+ 'http://www.mojvideo.com/playerapi.php?v=%s&t=1' % video_id, display_id)
+
+ if '<error>true</error>' in playerapi:
+ error_desc = self._html_search_regex(
+ r'<errordesc>([^<]*)</errordesc>', playerapi, 'error description', fatal=False)
+ raise ExtractorError('%s said: %s' % (self.IE_NAME, error_desc), expected=True)
+
+ title = self._html_search_regex(
+ r'<title>([^<]+)</title>', playerapi, 'title')
+ video_url = self._html_search_regex(
+ r'<file>([^<]+)</file>', playerapi, 'video URL')
+ thumbnail = self._html_search_regex(
+ r'<preview>([^<]+)</preview>', playerapi, 'thumbnail', fatal=False)
+ duration = parse_duration(self._html_search_regex(
+ r'<duration>([^<]+)</duration>', playerapi, 'duration', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'url': video_url,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ }
\ No newline at end of file
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ compat_str,
+ clean_html,
+)
+
+
+class MovieClipsIE(InfoExtractor):
+ _VALID_URL = r'https?://movieclips\.com/(?P<id>[\da-zA-Z]+)(?:-(?P<display_id>[\da-z-]+))?'
+ _TEST = {
+ 'url': 'http://movieclips.com/Wy7ZU-my-week-with-marilyn-movie-do-you-love-me/',
+ 'info_dict': {
+ 'id': 'Wy7ZU',
+ 'display_id': 'my-week-with-marilyn-movie-do-you-love-me',
+ 'ext': 'mp4',
+ 'title': 'My Week with Marilyn - Do You Love Me?',
+ 'description': 'md5:e86795bd332fe3cff461e7c8dc542acb',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+ show_id = display_id or video_id
+
+ config = self._download_xml(
+ 'http://config.movieclips.com/player/config/%s' % video_id,
+ show_id, 'Downloading player config')
+
+ if config.find('./country-region').text == 'false':
+ raise ExtractorError(
+ '%s said: %s' % (self.IE_NAME, config.find('./region_alert').text), expected=True)
+
+ properties = config.find('./video/properties')
+ smil_file = properties.attrib['smil_file']
+
+ smil = self._download_xml(smil_file, show_id, 'Downloading SMIL')
+ base_url = smil.find('./head/meta').attrib['base']
+
+ formats = []
+ for video in smil.findall('./body/switch/video'):
+ vbr = int(video.attrib['system-bitrate']) / 1000
+ src = video.attrib['src']
+ formats.append({
+ 'url': base_url,
+ 'play_path': src,
+ 'ext': src.split(':')[0],
+ 'vbr': vbr,
+ 'format_id': '%dk' % vbr,
+ })
+
+ self._sort_formats(formats)
+
+ title = '%s - %s' % (properties.attrib['clip_movie_title'], properties.attrib['clip_title'])
+ description = clean_html(compat_str(properties.attrib['clip_description']))
+ thumbnail = properties.attrib['image']
+ categories = properties.attrib['clip_categories'].split(',')
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'categories': categories,
+ 'formats': formats,
+ }
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ unified_strdate,
+)
+
+
+class MusicVaultIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.musicvault\.com/(?P<uploader_id>[^/?#]*)/video/(?P<display_id>[^/?#]*)_(?P<id>[0-9]+)\.html'
+ _TEST = {
+ 'url': 'http://www.musicvault.com/the-allman-brothers-band/video/straight-from-the-heart_1010863.html',
+ 'md5': '2cdbb3ae75f7fb3519821507d2fb3c15',
+ 'info_dict': {
+ 'id': '1010863',
+ 'ext': 'mp4',
+ 'uploader_id': 'the-allman-brothers-band',
+ 'title': 'Straight from the Heart',
+ 'duration': 244,
+ 'uploader': 'The Allman Brothers Band',
+ 'thumbnail': 're:^https?://.*/thumbnail/.*',
+ 'upload_date': '19811216',
+ 'location': 'Capitol Theatre (Passaic, NJ)',
+ 'description': 'Listen to The Allman Brothers Band perform Straight from the Heart at Capitol Theatre (Passaic, NJ) on Dec 16, 1981',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('display_id')
+ webpage = self._download_webpage(url, display_id)
+
+ thumbnail = self._search_regex(
+ r'<meta itemprop="thumbnail" content="([^"]+)"',
+ webpage, 'thumbnail', fatal=False)
+
+ data_div = self._search_regex(
+ r'(?s)<div class="data">(.*?)</div>', webpage, 'data fields')
+ uploader = self._html_search_regex(
+ r'<h1.*?>(.*?)</h1>', data_div, 'uploader', fatal=False)
+ title = self._html_search_regex(
+ r'<h2.*?>(.*?)</h2>', data_div, 'title')
+ upload_date = unified_strdate(self._html_search_regex(
+ r'<h3.*?>(.*?)</h3>', data_div, 'uploader', fatal=False))
+ location = self._html_search_regex(
+ r'<h4.*?>(.*?)</h4>', data_div, 'location', fatal=False)
+
+ duration = parse_duration(self._html_search_meta('duration', webpage))
+
+ VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
+ kaltura_id = self._search_regex(
+ r'<div id="video-detail-player" data-kaltura-id="([^"]+)"',
+ webpage, 'kaltura ID')
+ video_url = VIDEO_URL_TEMPLATE % {
+ 'entry_id': kaltura_id,
+ 'wid': self._search_regex(r'/wid/_([0-9]+)/', webpage, 'wid'),
+ 'uid': self._search_regex(r'uiconf_id/([0-9]+)/', webpage, 'uid'),
+ }
+
+ return {
+ 'id': mobj.group('id'),
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'display_id': display_id,
+ 'uploader_id': mobj.group('uploader_id'),
+ 'thumbnail': thumbnail,
+ 'description': self._html_search_meta('description', webpage),
+ 'upload_date': upload_date,
+ 'location': location,
+ 'title': title,
+ 'uploader': uploader,
+ 'duration': duration,
+ }
import re
from .common import InfoExtractor
+from ..utils import (
+ remove_end,
+ parse_duration,
+)
class NBAIE(InfoExtractor):
_VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)(?:/index\.html)?(?:\?.*)?$'
_TEST = {
'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
- 'md5': u'c0edcfc37607344e2ff8f13c378c88a4',
+ 'md5': 'c0edcfc37607344e2ff8f13c378c88a4',
'info_dict': {
'id': '0021200253-okc-bkn-recap.nba',
'ext': 'mp4',
- 'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
'title': 'Thunder vs. Nets',
+ 'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
+ 'duration': 181,
},
}
video_url = 'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
shortened_video_id = video_id.rpartition('/')[2]
- title = self._og_search_title(webpage, default=shortened_video_id).replace('NBA.com: ', '')
+ title = remove_end(
+ self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com')
+
+ description = self._og_search_description(webpage)
+ duration = parse_duration(
+ self._html_search_meta('duration', webpage, 'duration', fatal=False))
- description = self._html_search_regex(r'<meta name="description" (?:content|value)="(.*?)" />', webpage, 'description', fatal=False)
return {
'id': shortened_video_id,
'url': video_url,
'title': title,
'description': description,
+ 'duration': duration,
}
import json
from .common import InfoExtractor
-from ..utils import find_xpath_attr, compat_str
+from ..utils import (
+ compat_str,
+ ExtractorError,
+ find_xpath_attr,
+)
class NBCIE(InfoExtractor):
flags=re.MULTILINE)
bootstrap = json.loads(bootstrap_json)
info = bootstrap['results'][0]['video']
- playlist_url = info['fallbackPlaylistUrl'] + '?form=MPXNBCNewsAPI'
mpxid = info['mpxId']
- all_videos = self._download_json(playlist_url, title)['videos']
- # The response contains additional videos
- info = next(v for v in all_videos if v['mpxId'] == mpxid)
+
+ base_urls = [
+ info['fallbackPlaylistUrl'],
+ info['associatedPlaylistUrl'],
+ ]
+
+ for base_url in base_urls:
+ playlist_url = base_url + '?form=MPXNBCNewsAPI'
+ all_videos = self._download_json(playlist_url, title)['videos']
+
+ try:
+ info = next(v for v in all_videos if v['mpxId'] == mpxid)
+ break
+ except StopIteration:
+ continue
+
+ if info is None:
+ raise ExtractorError('Could not find video in playlists')
return {
'_type': 'url',
+from __future__ import unicode_literals
+
import re
import json
'path': initial_video_url.replace('.mp4', '_sd.mp4'),
})
path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
- path_doc = self._download_xml(path_url, video_id,
- u'Downloading final video url')
+ path_doc = self._download_xml(
+ path_url, video_id, 'Downloading final video url')
video_url = path_doc.find('path').text
join = compat_urlparse.urljoin
class NHLIE(NHLBaseInfoExtractor):
- IE_NAME = u'nhl.com'
- _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console\?.*?(?<=[?&])id=(?P<id>\d+)'
+ IE_NAME = 'nhl.com'
+ _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console\?.*?(?:[?&])id=(?P<id>[0-9]+)'
_TEST = {
- u'url': u'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
- u'file': u'453614.mp4',
- u'info_dict': {
- u'title': u'Quick clip: Weise 4-3 goal vs Flames',
- u'description': u'Dale Weise scores his first of the season to put the Canucks up 4-3.',
- u'duration': 18,
- u'upload_date': u'20131006',
+ 'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
+ 'info_dict': {
+ 'id': '453614',
+ 'ext': 'mp4',
+ 'title': 'Quick clip: Weise 4-3 goal vs Flames',
+ 'description': 'Dale Weise scores his first of the season to put the Canucks up 4-3.',
+ 'duration': 18,
+ 'upload_date': '20131006',
},
}
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
- info_json = self._download_webpage(json_url, video_id,
- u'Downloading info json')
- info_json = self._fix_json(info_json)
- info = json.loads(info_json)[0]
- return self._extract_video(info)
+ data = self._download_json(
+ json_url, video_id, transform_source=self._fix_json)
+ return self._extract_video(data[0])
class NHLVideocenterIE(NHLBaseInfoExtractor):
- IE_NAME = u'nhl.com:videocenter'
- IE_DESC = u'NHL videocenter category'
- _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?.*?catid=(?P<catid>[^&]+))?'
-
- @classmethod
- def suitable(cls, url):
- if NHLIE.suitable(url):
- return False
- return super(NHLVideocenterIE, cls).suitable(url)
+ IE_NAME = 'nhl.com:videocenter'
+ IE_DESC = 'NHL videocenter category'
+ _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?.*?catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
+ _TEST = {
+ 'url': 'http://video.canucks.nhl.com/videocenter/console?catid=999',
+ 'info_dict': {
+ 'id': '999',
+ 'title': 'Highlights',
+ },
+ 'playlist_count': 12,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
cat_id = self._search_regex(
[r'var defaultCatId = "(.+?)";',
r'{statusIndex:0,index:0,.*?id:(.*?),'],
- webpage, u'category id')
+ webpage, 'category id')
playlist_title = self._html_search_regex(
r'tab0"[^>]*?>(.*?)</td>',
- webpage, u'playlist title', flags=re.DOTALL).lower().capitalize()
+ webpage, 'playlist title', flags=re.DOTALL).lower().capitalize()
data = compat_urllib_parse.urlencode({
'cid': cat_id,
response = self._fix_json(response)
if not response.strip():
self._downloader.report_warning(u'Got an empty reponse, trying '
- u'adding the "newvideos" parameter')
+ 'adding the "newvideos" parameter')
response = self._download_webpage(request_url + '&newvideos=true',
playlist_title)
response = self._fix_json(response)
'_type': 'playlist',
'title': playlist_title,
'id': cat_id,
- 'entries': [self._extract_video(i) for i in videos],
+ 'entries': [self._extract_video(v) for v in videos],
}
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_parse,
+ compat_urllib_request,
+ xpath_with_ns,
+)
+
+_x = lambda p: xpath_with_ns(p, {'xspf': 'http://xspf.org/ns/0/'})
+_find = lambda el, p: el.find(_x(p)).text.strip()
+
+
+class NosVideoIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?nosvideo\.com/' + \
+ '(?:embed/|\?v=)(?P<id>[A-Za-z0-9]{12})/?'
+ _PLAYLIST_URL = 'http://nosvideo.com/xml/{xml_id:s}.xml'
+ _TEST = {
+ 'url': 'http://nosvideo.com/?v=drlp6s40kg54',
+ 'md5': '4b4ac54c6ad5d70ab88f2c2c6ccec71c',
+ 'info_dict': {
+ 'id': 'drlp6s40kg54',
+ 'ext': 'mp4',
+ 'title': 'big_buck_bunny_480p_surround-fix.avi.mp4',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ fields = {
+ 'id': video_id,
+ 'op': 'download1',
+ 'method_free': 'Continue to Video',
+ }
+ post = compat_urllib_parse.urlencode(fields)
+ req = compat_urllib_request.Request(url, post)
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+ webpage = self._download_webpage(req, video_id,
+ 'Downloading download page')
+ xml_id = self._search_regex(r'php\|([^\|]+)\|', webpage, 'XML ID')
+ playlist_url = self._PLAYLIST_URL.format(xml_id=xml_id)
+ playlist = self._download_xml(playlist_url, video_id)
+
+ track = playlist.find(_x('.//xspf:track'))
+ title = _find(track, './xspf:title')
+ url = _find(track, './xspf:file')
+ thumbnail = _find(track, './xspf:image')
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': url,
+ }]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
+# encoding: utf-8
from __future__ import unicode_literals
import re
class NownessIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?nowness\.com/[^?#]*?/(?P<id>[0-9]+)/(?P<slug>[^/]+?)(?:$|[?#])'
-
- _TEST = {
- 'url': 'http://www.nowness.com/day/2013/6/27/3131/candor--the-art-of-gesticulation',
- 'md5': '068bc0202558c2e391924cb8cc470676',
- 'info_dict': {
- 'id': '2520295746001',
- 'ext': 'mp4',
- 'description': 'Candor: The Art of Gesticulation',
- 'uploader': 'Nowness',
- 'title': 'Candor: The Art of Gesticulation',
- }
- }
+ _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/[^?#]*?/(?P<id>[0-9]+)/(?P<slug>[^/]+?)(?:$|[?#])'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.nowness.com/day/2013/6/27/3131/candor--the-art-of-gesticulation',
+ 'md5': '068bc0202558c2e391924cb8cc470676',
+ 'info_dict': {
+ 'id': '2520295746001',
+ 'ext': 'mp4',
+ 'title': 'Candor: The Art of Gesticulation',
+ 'description': 'Candor: The Art of Gesticulation',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'uploader': 'Nowness',
+ }
+ },
+ {
+ 'url': 'http://cn.nowness.com/day/2014/8/7/4069/kasper-bj-rke-ft-jaakko-eino-kalevi--tnr',
+ 'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
+ 'info_dict': {
+ 'id': '3716354522001',
+ 'ext': 'mp4',
+ 'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
+ 'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'uploader': 'Nowness',
+ }
+ },
+ ]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
from .common import InfoExtractor
from ..utils import (
unified_strdate,
+ qualities,
)
'md5': '4b3f9c429157ec4775f2c9cb7b911016',
'info_dict': {
'id': 'VPWON_1220719',
- 'ext': 'mp4',
+ 'ext': 'm4v',
'title': 'Nieuwsuur',
'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
'upload_date': '20140622',
'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
video_id,
# We have to remove the javascript callback
- transform_source=lambda j: re.sub(r'parseMetadata\((.*?)\);\n//epc', r'\1', j)
+ transform_source=lambda j: re.sub(r'parseMetadata\((.*?)\);\n//.*$', r'\1', j)
)
token_page = self._download_webpage(
'http://ida.omroep.nl/npoplayer/i.js',
video_id,
note='Downloading token'
)
- token = self._search_regex(r'npoplayer.token = "(.+?)"', token_page, 'token')
- streams_info = self._download_json(
- 'http://ida.omroep.nl/odi/?prid=%s&puboptions=h264_std&adaptive=yes&token=%s' % (video_id, token),
- video_id
- )
+ token = self._search_regex(r'npoplayer\.token = "(.+?)"', token_page, 'token')
- stream_info = self._download_json(
- streams_info['streams'][0] + '&type=json',
- video_id,
- 'Downloading stream info'
- )
+ formats = []
+ quality = qualities(['adaptive', 'h264_sb', 'h264_bb', 'h264_std'])
+ for format_id in metadata['pubopties']:
+ streams_info = self._download_json(
+ 'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s' % (video_id, format_id, token),
+ video_id, 'Downloading %s streams info' % format_id)
+ stream_info = self._download_json(
+ streams_info['streams'][0] + '&type=json',
+ video_id, 'Downloading %s stream info' % format_id)
+ if format_id == 'adaptive':
+ formats.extend(self._extract_m3u8_formats(stream_info['url'], video_id))
+ else:
+ formats.append({
+ 'url': stream_info['url'],
+ 'format_id': format_id,
+ 'quality': quality(format_id),
+ })
+ self._sort_formats(formats)
return {
'id': video_id,
'title': metadata['titel'],
- 'ext': 'mp4',
- 'url': stream_info['url'],
'description': metadata['info'],
'thumbnail': metadata['images'][-1]['url'],
'upload_date': unified_strdate(metadata['gidsdatum']),
+ 'formats': formats,
}
webpage = self._download_webpage(
request, video_id, 'Downloading %s page' % format_id)
video_url = self._html_search_regex(
- r'<a href="([^"]+)"\s*>Continue to watch video', webpage, '%s video URL' % format_id, fatal=False)
+ r'<a\s+href="([^"]+)"\s+class="b_link">', webpage, '%s video URL' % format_id, fatal=False)
if not video_url:
continue
formats.append({
webpage = self._download_webpage(
'http://m.nuvid.com/video/%s' % video_id, video_id, 'Downloading video page')
title = self._html_search_regex(
- r'<div class="title">\s+<h2[^>]*>([^<]+)</h2>', webpage, 'title').strip()
- thumbnail = self._html_search_regex(
- r'href="(/thumbs/[^"]+)"[^>]*data-link_type="thumbs"',
- webpage, 'thumbnail URL', fatal=False)
+ [r'<span title="([^"]+)">',
+ r'<div class="thumb-holder video">\s*<h5[^>]*>([^<]+)</h5>'], webpage, 'title').strip()
+ thumbnails = [
+ {
+ 'url': thumb_url,
+ } for thumb_url in re.findall(r'<img src="([^"]+)" alt="" />', webpage)
+ ]
+ thumbnail = thumbnails[0]['url'] if thumbnails else None
duration = parse_duration(self._html_search_regex(
- r'Length:\s*<span>(\d{2}:\d{2})</span>',webpage, 'duration', fatal=False))
+ r'<i class="fa fa-clock-o"></i>\s*(\d{2}:\d{2})', webpage, 'duration', fatal=False))
upload_date = unified_strdate(self._html_search_regex(
- r'Added:\s*<span>(\d{4}-\d{2}-\d{2})</span>', webpage, 'upload date', fatal=False))
+ r'<i class="fa fa-user"></i>\s*(\d{4}-\d{2}-\d{2})', webpage, 'upload date', fatal=False))
return {
'id': video_id,
'title': title,
- 'thumbnail': 'http://m.nuvid.com%s' % thumbnail,
+ 'thumbnails': thumbnails,
+ 'thumbnail': thumbnail,
'duration': duration,
'upload_date': upload_date,
'age_limit': 18,
+++ /dev/null
-# coding: utf-8
-from __future__ import unicode_literals
-
-import calendar
-import datetime
-import re
-
-from .common import InfoExtractor
-
-# audios on oe1.orf.at are only available for 7 days, so we can't
-# add tests.
-
-
-class OE1IE(InfoExtractor):
- IE_DESC = 'oe1.orf.at'
- _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
-
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- show_id = mobj.group('id')
-
- data = self._download_json(
- 'http://oe1.orf.at/programm/%s/konsole' % show_id,
- show_id
- )
-
- timestamp = datetime.datetime.strptime('%s %s' % (
- data['item']['day_label'],
- data['item']['time']
- ), '%d.%m.%Y %H:%M')
- unix_timestamp = calendar.timegm(timestamp.utctimetuple())
-
- return {
- 'id': show_id,
- 'title': data['item']['title'],
- 'url': data['item']['url_stream'],
- 'ext': 'mp3',
- 'description': data['item'].get('info'),
- 'timestamp': unix_timestamp
- }
import json
from .common import InfoExtractor
-from ..utils import unescapeHTML
+from ..utils import (
+ unescapeHTML,
+ ExtractorError,
+)
class OoyalaIE(InfoExtractor):
_VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)'
- _TEST = {
- # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video
- 'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
- 'md5': '3f5cceb3a7bf461d6c29dc466cf8033c',
- 'info_dict': {
- 'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
- 'ext': 'mp4',
- 'title': 'Explaining Data Recovery from Hard Drives and SSDs',
- 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
+ _TESTS = [
+ {
+ # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video
+ 'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
+ 'md5': '3f5cceb3a7bf461d6c29dc466cf8033c',
+ 'info_dict': {
+ 'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
+ 'ext': 'mp4',
+ 'title': 'Explaining Data Recovery from Hard Drives and SSDs',
+ 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
+ },
+ }, {
+ # Only available for ipad
+ 'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
+ 'md5': '4b9754921fddb68106e48c142e2a01e6',
+ 'info_dict': {
+ 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
+ 'ext': 'mp4',
+ 'title': 'Simulation Overview - Levels of Simulation',
+ 'description': '',
+ },
},
- }
+ ]
@staticmethod
def _url_for_embed_code(embed_code):
player = self._download_webpage(player_url, embedCode)
mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
player, 'mobile player url')
- mobile_player = self._download_webpage(mobile_url, embedCode)
- videos_info = self._search_regex(
- r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
- mobile_player, 'info').replace('\\"','"')
- videos_more_info = self._search_regex(r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"','"')
+ # Looks like some videos are only available for particular devices
+ # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
+ # is only available for ipad)
+ # Working around with fetching URLs for all the devices found starting with 'unknown'
+ # until we succeed or eventually fail for each device.
+ devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
+ devices.remove('unknown')
+ devices.insert(0, 'unknown')
+ for device in devices:
+ mobile_player = self._download_webpage(
+ '%s&device=%s' % (mobile_url, device), embedCode,
+ 'Downloading mobile player JS for %s device' % device)
+ videos_info = self._search_regex(
+ r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
+ mobile_player, 'info', fatal=False, default=None)
+ if videos_info:
+ break
+ if not videos_info:
+ raise ExtractorError('Unable to extract info')
+ videos_info = videos_info.replace('\\"', '"')
+ videos_more_info = self._search_regex(
+ r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
videos_info = json.loads(videos_info)
- videos_more_info =json.loads(videos_more_info)
+ videos_more_info = json.loads(videos_more_info)
if videos_more_info.get('lineup'):
videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
import json
import re
+import calendar
+import datetime
from .common import InfoExtractor
from ..utils import (
)
-class ORFIE(InfoExtractor):
+class ORFTVthekIE(InfoExtractor):
+ IE_NAME = 'orf:tvthek'
+ IE_DESC = 'ORF TVthek'
_VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
_TEST = {
'entries': entries,
'id': playlist_id,
}
+
+
+# Audios on ORF radio are only available for 7 days, so we can't add tests.
+
+
+class ORFOE1IE(InfoExtractor):
+ IE_NAME = 'orf:oe1'
+ IE_DESC = 'Radio Österreich 1'
+ _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ show_id = mobj.group('id')
+
+ data = self._download_json(
+ 'http://oe1.orf.at/programm/%s/konsole' % show_id,
+ show_id
+ )
+
+ timestamp = datetime.datetime.strptime('%s %s' % (
+ data['item']['day_label'],
+ data['item']['time']
+ ), '%d.%m.%Y %H:%M')
+ unix_timestamp = calendar.timegm(timestamp.utctimetuple())
+
+ return {
+ 'id': show_id,
+ 'title': data['item']['title'],
+ 'url': data['item']['url_stream'],
+ 'ext': 'mp3',
+ 'description': data['item'].get('info'),
+ 'timestamp': unix_timestamp
+ }
+
+
+class ORFFM4IE(InfoExtractor):
+ IE_DESC = 'orf:fm4'
+ IE_DESC = 'radio FM4'
+ _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ show_date = mobj.group('date')
+ show_id = mobj.group('show')
+
+ data = self._download_json(
+ 'http://audioapi.orf.at/fm4/json/2.0/broadcasts/%s/4%s' % (show_date, show_id),
+ show_id
+ )
+
+ def extract_entry_dict(info, title, subtitle):
+ return {
+ 'id': info['loopStreamId'].replace('.mp3', ''),
+ 'url': 'http://loopstream01.apa.at/?channel=fm4&id=%s' % info['loopStreamId'],
+ 'title': title,
+ 'description': subtitle,
+ 'duration': (info['end'] - info['start']) / 1000,
+ 'timestamp': info['start'] / 1000,
+ 'ext': 'mp3'
+ }
+
+ entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]
+
+ return {
+ '_type': 'playlist',
+ 'id': show_id,
+ 'title': data['title'],
+ 'description': data['subtitle'],
+ 'entries': entries
+ }
\ No newline at end of file
--- /dev/null
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ js_to_json,
+)
+
+
+class PatreonIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)'
+ _TESTS = [
+ {
+ 'url': 'http://www.patreon.com/creation?hid=743933',
+ 'md5': 'e25505eec1053a6e6813b8ed369875cc',
+ 'info_dict': {
+ 'id': '743933',
+ 'ext': 'mp3',
+ 'title': 'Episode 166: David Smalley of Dogma Debate',
+ 'uploader': 'Cognitive Dissonance Podcast',
+ 'thumbnail': 're:^https?://.*$',
+ },
+ },
+ {
+ 'url': 'http://www.patreon.com/creation?hid=754133',
+ 'md5': '3eb09345bf44bf60451b8b0b81759d0a',
+ 'info_dict': {
+ 'id': '754133',
+ 'ext': 'mp3',
+ 'title': 'CD 167 Extra',
+ 'uploader': 'Cognitive Dissonance Podcast',
+ 'thumbnail': 're:^https?://.*$',
+ },
+ },
+ ]
+
+ # Currently Patreon exposes download URL via hidden CSS, so login is not
+ # needed. Keeping this commented for when this inevitably changes.
+ '''
+ def _login(self):
+ (username, password) = self._get_login_info()
+ if username is None:
+ return
+
+ login_form = {
+ 'redirectUrl': 'http://www.patreon.com/',
+ 'email': username,
+ 'password': password,
+ }
+
+ request = compat_urllib_request.Request(
+ 'https://www.patreon.com/processLogin',
+ compat_urllib_parse.urlencode(login_form).encode('utf-8')
+ )
+ login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+
+ if re.search(r'onLoginFailed', login_page):
+ raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
+
+ def _real_initialize(self):
+ self._login()
+ '''
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group(1)
+
+ webpage = self._download_webpage(url, video_id)
+ title = self._og_search_title(webpage).strip()
+
+ attach_fn = self._html_search_regex(
+ r'<div class="attach"><a target="_blank" href="([^"]+)">',
+ webpage, 'attachment URL', default=None)
+ if attach_fn is not None:
+ video_url = 'http://www.patreon.com' + attach_fn
+ thumbnail = self._og_search_thumbnail(webpage)
+ uploader = self._html_search_regex(
+ r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
+ else:
+ playlist_js = self._search_regex(
+ r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',
+ webpage, 'playlist JSON')
+ playlist_json = js_to_json(playlist_js)
+ playlist = json.loads(playlist_json)
+ data = playlist[0]
+ video_url = self._proto_relative_url(data['mp3'])
+ thumbnail = self._proto_relative_url(data.get('cover'))
+ uploader = data.get('artist')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp3',
+ 'title': title,
+ 'uploader': uploader,
+ 'thumbnail': thumbnail,
+ }
)
'''
- _TEST = {
- 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
- 'md5': 'ce1888486f0908d555a8093cac9a7362',
- 'info_dict': {
- 'id': '2365006249',
- 'ext': 'mp4',
- 'title': 'A More Perfect Union',
- 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
- 'duration': 3190,
+ _TESTS = [
+ {
+ 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
+ 'md5': 'ce1888486f0908d555a8093cac9a7362',
+ 'info_dict': {
+ 'id': '2365006249',
+ 'ext': 'mp4',
+ 'title': 'A More Perfect Union',
+ 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
+ 'duration': 3190,
+ },
+ },
+ {
+ 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
+ 'md5': '143c98aa54a346738a3d78f54c925321',
+ 'info_dict': {
+ 'id': '2365297690',
+ 'ext': 'mp4',
+ 'title': 'Losing Iraq',
+ 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+ 'duration': 5050,
+ },
+ },
+ {
+ 'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
+ 'md5': 'b19856d7f5351b17a5ab1dc6a64be633',
+ 'info_dict': {
+ 'id': '2201174722',
+ 'ext': 'mp4',
+ 'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist',
+ 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
+ 'duration': 801,
+ },
},
- }
+ {
+ 'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/',
+ 'md5': 'c62859342be2a0358d6c9eb306595978',
+ 'info_dict': {
+ 'id': '2365297708',
+ 'ext': 'mp4',
+ 'description': 'md5:68d87ef760660eb564455eb30ca464fe',
+ 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
+ 'duration': 6559,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+ ]
- def _real_extract(self, url):
+ def _extract_ids(self, url):
mobj = re.match(self._VALID_URL, url)
presumptive_id = mobj.group('presumptive_id')
display_id = presumptive_id
if presumptive_id:
webpage = self._download_webpage(url, display_id)
+
+ MEDIA_ID_REGEXES = [
+ r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed
+ r'class="coveplayerid">([^<]+)<', # coveplayer
+ ]
+
+ media_id = self._search_regex(
+ MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
+ if media_id:
+ return media_id, presumptive_id
+
url = self._search_regex(
- r'<iframe\s+id=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
+ r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
webpage, 'player URL')
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
display_id = video_id
+ return video_id, display_id
+
+ def _real_extract(self, url):
+ video_id, display_id = self._extract_ids(url)
+
info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
info = self._download_json(info_url, display_id)
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_parse,
+ compat_urllib_request,
+ ExtractorError,
+ float_or_none,
+ int_or_none,
+)
+
+
+class PlayFMIE(InfoExtractor):
+ IE_NAME = 'play.fm'
+ _VALID_URL = r'https?://(?:www\.)?play\.fm/[^?#]*(?P<upload_date>[0-9]{8})(?P<id>[0-9]{6})(?:$|[?#])'
+
+ _TEST = {
+ 'url': 'http://www.play.fm/recording/leipzigelectronicmusicbatofarparis_fr20140712137220',
+ 'md5': 'c505f8307825a245d0c7ad1850001f22',
+ 'info_dict': {
+ 'id': '137220',
+ 'ext': 'mp3',
+ 'title': 'LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
+ 'uploader': 'Sven Tasnadi',
+ 'uploader_id': 'sventasnadi',
+ 'duration': 5627.428,
+ 'upload_date': '20140712',
+ 'view_count': int,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ upload_date = mobj.group('upload_date')
+
+ rec_data = compat_urllib_parse.urlencode({'rec_id': video_id})
+ req = compat_urllib_request.Request(
+ 'http://www.play.fm/flexRead/recording', data=rec_data)
+ req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ rec_doc = self._download_xml(req, video_id)
+
+ error_node = rec_doc.find('./error')
+ if error_node is not None:
+ raise ExtractorError('An error occured: %s (code %s)' % (
+ error_node.text, rec_doc.find('./status').text))
+
+ recording = rec_doc.find('./recording')
+ title = recording.find('./title').text
+ view_count = int_or_none(recording.find('./stats/playcount').text)
+ duration = float_or_none(recording.find('./duration').text, scale=1000)
+ thumbnail = recording.find('./image').text
+
+ artist = recording.find('./artists/artist')
+ uploader = artist.find('./name').text
+ uploader_id = artist.find('./slug').text
+
+ video_url = '%s//%s/%s/%s/offset/0/sh/%s/rec/%s/jingle/%s/loc/%s' % (
+ 'http:', recording.find('./url').text,
+ recording.find('./_class').text, recording.find('./file_id').text,
+ rec_doc.find('./uuid').text, video_id,
+ rec_doc.find('./jingle/file_id').text,
+ 'http%3A%2F%2Fwww.play.fm%2Fplayer',
+ )
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp3',
+ 'filesize': int_or_none(recording.find('./size').text),
+ 'title': title,
+ 'upload_date': upload_date,
+ 'view_count': view_count,
+ 'duration': duration,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ }
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
class PornotubeIE(InfoExtractor):
- _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
+ _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
_TEST = {
- u'url': u'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
- u'file': u'1689755.flv',
- u'md5': u'374dd6dcedd24234453b295209aa69b6',
- u'info_dict': {
- u"upload_date": u"20090708",
- u"title": u"Marilyn-Monroe-Bathing",
- u"age_limit": 18
+ 'url': 'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
+ 'md5': '374dd6dcedd24234453b295209aa69b6',
+ 'info_dict': {
+ 'id': '1689755',
+ 'ext': 'flv',
+ 'upload_date': '20090708',
+ 'title': 'Marilyn-Monroe-Bathing',
+ 'age_limit': 18
}
}
# Get the video URL
VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",'
- video_url = self._search_regex(VIDEO_URL_RE, webpage, u'video url')
+ video_url = self._search_regex(VIDEO_URL_RE, webpage, 'video url')
video_url = compat_urllib_parse.unquote(video_url)
#Get the uploaded date
VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
- upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False)
- if upload_date: upload_date = unified_strdate(upload_date)
+ upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, 'upload date', fatal=False)
+ if upload_date:
+ upload_date = unified_strdate(upload_date)
age_limit = self._rta_search(webpage)
- info = {'id': video_id,
- 'url': video_url,
- 'uploader': None,
- 'upload_date': upload_date,
- 'title': video_title,
- 'ext': 'flv',
- 'format': 'flv',
- 'age_limit': age_limit}
-
- return [info]
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'upload_date': upload_date,
+ 'title': video_title,
+ 'ext': 'flv',
+ 'format': 'flv',
+ 'age_limit': age_limit,
+ }
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ determine_ext,
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+
+
+class PromptFileIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?promptfile\.com/l/(?P<id>[0-9A-Z\-]+)'
+ _FILE_NOT_FOUND_REGEX = r'<div.+id="not_found_msg".+>.+</div>[^-]'
+ _TEST = {
+ 'url': 'http://www.promptfile.com/l/D21B4746E9-F01462F0FF',
+ 'md5': 'd1451b6302da7215485837aaea882c4c',
+ 'info_dict': {
+ 'id': 'D21B4746E9-F01462F0FF',
+ 'ext': 'mp4',
+ 'title': 'Birds.mp4',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+
+ if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
+ raise ExtractorError('Video %s does not exist' % video_id,
+ expected=True)
+
+ fields = dict(re.findall(r'''(?x)type="hidden"\s+
+ name="(.+?)"\s+
+ value="(.*?)"
+ ''', webpage))
+ post = compat_urllib_parse.urlencode(fields)
+ req = compat_urllib_request.Request(url, post)
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+ webpage = self._download_webpage(
+ req, video_id, 'Downloading video page')
+
+ url = self._html_search_regex(r'url:\s*\'([^\']+)\'', webpage, 'URL')
+ title = self._html_search_regex(
+ r'<span.+title="([^"]+)">', webpage, 'title')
+ thumbnail = self._html_search_regex(
+ r'<div id="player_overlay">.*button>.*?<img src="([^"]+)"',
+ webpage, 'thumbnail', fatal=False, flags=re.DOTALL)
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': url,
+ 'ext': determine_ext(title),
+ }]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
r'<h1 class="videoTitle[^"]*">(.+?)</h1>',
webpage, u'title')
- video_thumbnail = self._html_search_regex(
- r'playerInnerHTML.+?<img\s+src="(.+?)"',
- webpage, u'thumbnail', fatal=False)
+ video_thumbnail = self._og_search_thumbnail(webpage)
# No self-labeling, but they describe themselves as
# "Home of Videos Porno"
from __future__ import unicode_literals
import re
-import time
from .common import InfoExtractor
-from ..utils import strip_jsonp
+from ..utils import str_or_none
class ReverbNationIE(InfoExtractor):
_VALID_URL = r'^https?://(?:www\.)?reverbnation\.com/.*?/song/(?P<id>\d+).*?$'
_TESTS = [{
'url': 'http://www.reverbnation.com/alkilados/song/16965047-mona-lisa',
- 'file': '16965047.mp3',
'md5': '3da12ebca28c67c111a7f8b262d3f7a7',
'info_dict': {
+ "id": "16965047",
+ "ext": "mp3",
"title": "MONA LISA",
"uploader": "ALKILADOS",
- "uploader_id": 216429,
- "thumbnail": "//gp1.wac.edgecastcdn.net/802892/production_public/Photo/13761700/image/1366002176_AVATAR_MONA_LISA.jpg"
+ "uploader_id": "216429",
+ "thumbnail": "re:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$"
},
}]
song_id = mobj.group('id')
api_res = self._download_json(
- 'https://api.reverbnation.com/song/%s?callback=api_response_5&_=%d'
- % (song_id, int(time.time() * 1000)),
+ 'https://api.reverbnation.com/song/%s' % song_id,
song_id,
- transform_source=strip_jsonp,
note='Downloading information of song %s' % song_id
)
'title': api_res.get('name'),
'url': api_res.get('url'),
'uploader': api_res.get('artist', {}).get('name'),
- 'uploader_id': api_res.get('artist', {}).get('id'),
- 'thumbnail': api_res.get('image', api_res.get('thumbnail')),
+ 'uploader_id': str_or_none(api_res.get('artist', {}).get('id')),
+ 'thumbnail': self._proto_relative_url(
+ api_res.get('image', api_res.get('thumbnail'))),
'ext': 'mp3',
'vcodec': 'none',
}
page = self._download_webpage('https://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
data = json.loads(self._html_search_regex(
- r'<div class="js-player-embed" data-video="([^"]+)"', page, 'data video'))['data']
+ r'<div class="js-player-embed(?: player-embed)?" data-video="([^"]+)"', page, 'data video'))['data']
video_url = data.get('downloadUrl') or data.get('url')
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_duration
+
+
+class RtlXlIE(InfoExtractor):
+ IE_NAME = 'rtlxl.nl'
+ _VALID_URL = r'https?://www\.rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
+
+ _TEST = {
+ 'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
+ 'md5': 'cc16baa36a6c169391f0764fa6b16654',
+ 'info_dict': {
+ 'id': '6e4203a6-0a5e-3596-8424-c599a59e0677',
+ 'ext': 'mp4',
+ 'title': 'RTL Nieuws - Laat',
+ 'description': 'md5:6b61f66510c8889923b11f2778c72dc5',
+ 'timestamp': 1408051800,
+ 'upload_date': '20140814',
+ 'duration': 576.880,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ uuid = mobj.group('uuid')
+
+ info = self._download_json(
+ 'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
+ uuid)
+
+ material = info['material'][0]
+ episode_info = info['episodes'][0]
+
+ progname = info['abstracts'][0]['name']
+ subtitle = material['title'] or info['episodes'][0]['name']
+
+ videopath = material['videopath']
+ f4m_url = 'http://manifest.us.rtl.nl' + videopath
+
+ formats = self._extract_f4m_formats(f4m_url, uuid)
+
+ video_urlpart = videopath.split('/flash/')[1][:-4]
+ PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
+
+ formats.extend([
+ {
+ 'url': PG_URL_TEMPLATE % ('a2m', video_urlpart),
+ 'format_id': 'pg-sd',
+ },
+ {
+ 'url': PG_URL_TEMPLATE % ('a3m', video_urlpart),
+ 'format_id': 'pg-hd',
+ }
+ ])
+
+ return {
+ 'id': uuid,
+ 'title': '%s - %s' % (progname, subtitle),
+ 'formats': formats,
+ 'timestamp': material['original_date'],
+ 'description': episode_info['synopsis'],
+ 'duration': parse_duration(material.get('duration')),
+ }
},
{
'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
- 'info_dict': {
- 'id': '153819',
- 'ext': 'flv',
- 'title': 'Deluxe - Alles was Spaß macht - Thema u.a.: Luxushotel für Vierbeiner',
- 'description': 'md5:c3705e1bb32e1a5b2bcd634fc065c631',
- 'thumbnail': 'http://autoimg.static-fra.de/ntvnow/383157/1500x1500/image2.jpg',
- 'upload_date': '20140221',
- 'duration': 2429,
- },
- 'skip': 'Only works from Germany',
+ 'only_matching': True,
},
]
# encoding: utf-8
from __future__ import unicode_literals
-import re
import base64
+import re
+import time
from .common import InfoExtractor
from ..utils import (
struct_unpack,
+ remove_end,
)
+def _decrypt_url(png):
+ encrypted_data = base64.b64decode(png)
+ text_index = encrypted_data.find(b'tEXt')
+ text_chunk = encrypted_data[text_index - 4:]
+ length = struct_unpack('!I', text_chunk[:4])[0]
+ # Use bytearray to get integers when iterating in both python 2.x and 3.x
+ data = bytearray(text_chunk[8:8 + length])
+ data = [chr(b) for b in data if b != 0]
+ hash_index = data.index('#')
+ alphabet_data = data[:hash_index]
+ url_data = data[hash_index + 1:]
+
+ alphabet = []
+ e = 0
+ d = 0
+ for l in alphabet_data:
+ if d == 0:
+ alphabet.append(l)
+ d = e = (e + 1) % 4
+ else:
+ d -= 1
+ url = ''
+ f = 0
+ e = 3
+ b = 1
+ for letter in url_data:
+ if f == 0:
+ l = int(letter) * 10
+ f = 1
+ else:
+ if e == 0:
+ l += int(letter)
+ url += alphabet[l]
+ e = (b + 3) % 4
+ f = 0
+ b += 1
+ else:
+ e -= 1
+
+ return url
+
+
+
class RTVEALaCartaIE(InfoExtractor):
IE_NAME = 'rtve.es:alacarta'
IE_DESC = 'RTVE a la carta'
_VALID_URL = r'http://www\.rtve\.es/alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
- _TEST = {
+ _TESTS = [{
'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
- 'md5': '18fcd45965bdd076efdb12cd7f6d7b9e',
+ 'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
'info_dict': {
'id': '2491869',
'ext': 'mp4',
'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
},
- }
-
- def _decrypt_url(self, png):
- encrypted_data = base64.b64decode(png)
- text_index = encrypted_data.find(b'tEXt')
- text_chunk = encrypted_data[text_index-4:]
- length = struct_unpack('!I', text_chunk[:4])[0]
- # Use bytearray to get integers when iterating in both python 2.x and 3.x
- data = bytearray(text_chunk[8:8+length])
- data = [chr(b) for b in data if b != 0]
- hash_index = data.index('#')
- alphabet_data = data[:hash_index]
- url_data = data[hash_index+1:]
-
- alphabet = []
- e = 0
- d = 0
- for l in alphabet_data:
- if d == 0:
- alphabet.append(l)
- d = e = (e + 1) % 4
- else:
- d -= 1
- url = ''
- f = 0
- e = 3
- b = 1
- for letter in url_data:
- if f == 0:
- l = int(letter)*10
- f = 1
- else:
- if e == 0:
- l += int(letter)
- url += alphabet[l]
- e = (b + 3) % 4
- f = 0
- b += 1
- else:
- e -= 1
-
- return url
+ }, {
+ 'note': 'Live stream',
+ 'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
+ 'info_dict': {
+ 'id': '1694255',
+ 'ext': 'flv',
+ 'title': 'TODO',
+ }
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id)['page']['items'][0]
png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id
png = self._download_webpage(png_url, video_id, 'Downloading url information')
- video_url = self._decrypt_url(png)
+ video_url = _decrypt_url(png)
return {
'id': video_id,
'title': info['title'],
'url': video_url,
- 'thumbnail': info['image'],
+ 'thumbnail': info.get('image'),
+ 'page_url': url,
+ }
+
+
+class RTVELiveIE(InfoExtractor):
+ IE_NAME = 'rtve.es:live'
+ IE_DESC = 'RTVE.es live streams'
+ _VALID_URL = r'http://www\.rtve\.es/(?:deportes/directo|noticias|television)/(?P<id>[a-zA-Z0-9-]+)'
+
+ _TESTS = [{
+ 'url': 'http://www.rtve.es/noticias/directo-la-1/',
+ 'info_dict': {
+ 'id': 'directo-la-1',
+ 'ext': 'flv',
+ 'title': 're:^La 1 de TVE [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
+ },
+ 'params': {
+ 'skip_download': 'live stream',
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ start_time = time.gmtime()
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ player_url = self._search_regex(
+ r'<param name="movie" value="([^"]+)"/>', webpage, 'player URL')
+ title = remove_end(self._og_search_title(webpage), ' en directo')
+ title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)
+
+ vidplayer_id = self._search_regex(
+ r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
+ png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
+ png = self._download_webpage(png_url, video_id, 'Downloading url information')
+ video_url = _decrypt_url(png)
+
+ return {
+ 'id': video_id,
+ 'ext': 'flv',
+ 'title': title,
+ 'url': video_url,
+ 'app': 'rtve-live-live?ovpfv=2.1.2',
+ 'player_url': player_url,
+ 'rtmp_live': True,
}
IE_NAME = 'rutube:channel'
IE_DESC = 'Rutube channels'
_VALID_URL = r'http://rutube\.ru/tags/video/(?P<id>\d+)'
+ _TESTS = [{
+ 'url': 'http://rutube.ru/tags/video/1800/',
+ 'info_dict': {
+ 'id': '1800',
+ },
+ 'playlist_mincount': 68,
+ }]
_PAGE_TEMPLATE = 'http://rutube.ru/api/tags/video/%s/?page=%s&format=json'
IE_NAME = 'rutube:movie'
IE_DESC = 'Rutube movies'
_VALID_URL = r'http://rutube\.ru/metainfo/tv/(?P<id>\d+)'
+ _TESTS = []
_MOVIE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/?format=json'
_PAGE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json'
IE_NAME = 'rutube:person'
IE_DESC = 'Rutube person videos'
_VALID_URL = r'http://rutube\.ru/video/person/(?P<id>\d+)'
+ _TESTS = [{
+ 'url': 'http://rutube.ru/video/person/313878/',
+ 'info_dict': {
+ 'id': '313878',
+ },
+ 'playlist_mincount': 37,
+ }]
_PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json'
--- /dev/null
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ unified_strdate,
+)
+
+
+class SapoIE(InfoExtractor):
+ IE_DESC = 'SAPO Vídeos'
+ _VALID_URL = r'https?://(?:(?:v2|www)\.)?videos\.sapo\.(?:pt|cv|ao|mz|tl)/(?P<id>[\da-zA-Z]{20})'
+
+ _TESTS = [
+ {
+ 'url': 'http://videos.sapo.pt/UBz95kOtiWYUMTA5Ghfi',
+ 'md5': '79ee523f6ecb9233ac25075dee0eda83',
+ 'note': 'SD video',
+ 'info_dict': {
+ 'id': 'UBz95kOtiWYUMTA5Ghfi',
+ 'ext': 'mp4',
+ 'title': 'Benfica - Marcas na Hitória',
+ 'description': 'md5:c9082000a128c3fd57bf0299e1367f22',
+ 'duration': 264,
+ 'uploader': 'tiago_1988',
+ 'upload_date': '20080229',
+ 'categories': ['benfica', 'cabral', 'desporto', 'futebol', 'geovanni', 'hooijdonk', 'joao', 'karel', 'lisboa', 'miccoli'],
+ },
+ },
+ {
+ 'url': 'http://videos.sapo.pt/IyusNAZ791ZdoCY5H5IF',
+ 'md5': '90a2f283cfb49193fe06e861613a72aa',
+ 'note': 'HD video',
+ 'info_dict': {
+ 'id': 'IyusNAZ791ZdoCY5H5IF',
+ 'ext': 'mp4',
+ 'title': 'Codebits VII - Report',
+ 'description': 'md5:6448d6fd81ce86feac05321f354dbdc8',
+ 'duration': 144,
+ 'uploader': 'codebits',
+ 'upload_date': '20140427',
+ 'categories': ['codebits', 'codebits2014'],
+ },
+ },
+ {
+ 'url': 'http://v2.videos.sapo.pt/yLqjzPtbTimsn2wWBKHz',
+ 'md5': 'e5aa7cc0bdc6db9b33df1a48e49a15ac',
+ 'note': 'v2 video',
+ 'info_dict': {
+ 'id': 'yLqjzPtbTimsn2wWBKHz',
+ 'ext': 'mp4',
+ 'title': 'Hipnose Condicionativa 4',
+ 'description': 'md5:ef0481abf8fb4ae6f525088a6dadbc40',
+ 'duration': 692,
+ 'uploader': 'sapozen',
+ 'upload_date': '20090609',
+ 'categories': ['condicionativa', 'heloisa', 'hipnose', 'miranda', 'sapo', 'zen'],
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ item = self._download_xml(
+ 'http://rd3.videos.sapo.pt/%s/rss2' % video_id, video_id).find('./channel/item')
+
+ title = item.find('./title').text
+ description = item.find('./{http://videos.sapo.pt/mrss/}synopse').text
+ thumbnail = item.find('./{http://search.yahoo.com/mrss/}content').get('url')
+ duration = parse_duration(item.find('./{http://videos.sapo.pt/mrss/}time').text)
+ uploader = item.find('./{http://videos.sapo.pt/mrss/}author').text
+ upload_date = unified_strdate(item.find('./pubDate').text)
+ view_count = int(item.find('./{http://videos.sapo.pt/mrss/}views').text)
+ comment_count = int(item.find('./{http://videos.sapo.pt/mrss/}comment_count').text)
+ tags = item.find('./{http://videos.sapo.pt/mrss/}tags').text
+ categories = tags.split() if tags else []
+ age_limit = 18 if item.find('./{http://videos.sapo.pt/mrss/}m18').text == 'true' else 0
+
+ video_url = item.find('./{http://videos.sapo.pt/mrss/}videoFile').text
+ video_size = item.find('./{http://videos.sapo.pt/mrss/}videoSize').text.split('x')
+
+ formats = [{
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'format_id': 'sd',
+ 'width': int(video_size[0]),
+ 'height': int(video_size[1]),
+ }]
+
+ if item.find('./{http://videos.sapo.pt/mrss/}HD').text == 'true':
+ formats.append({
+ 'url': re.sub(r'/mov/1$', '/mov/39', video_url),
+ 'ext': 'mp4',
+ 'format_id': 'hd',
+ 'width': 1280,
+ 'height': 720,
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ 'view_count': view_count,
+ 'comment_count': comment_count,
+ 'categories': categories,
+ 'age_limit': age_limit,
+ 'formats': formats,
+ }
'upload_date': '20120816',
'uploader': 'Howcast',
'uploader_id': 'Howcast',
- 'description': 'md5:4f0aac94361a12e1ce57d74f85265175',
+ 'description': 're:(?s).* Hi, my name is Rene Dreifuss\. And I\'m here to show you some MMA.*',
},
'params': {
'skip_download': True
--- /dev/null
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import json
+import re
+from .common import InfoExtractor
+from ..utils import (
+ js_to_json,
+ remove_end,
+)
+
+
+class SBSIE(InfoExtractor):
+ IE_DESC = 'sbs.com.au'
+ _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/single/(?P<id>[0-9]+)/'
+
+ _TESTS = [{
+ # Original URL is handled by the generic IE which finds the iframe:
+ # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation
+ 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed',
+ 'md5': '3150cf278965eeabb5b4cea1c963fe0a',
+ 'info_dict': {
+ 'id': '320403011771',
+ 'ext': 'flv',
+ 'title': 'Dingo Conservation',
+ 'description': 'Dingoes are on the brink of extinction; most of the animals we think are dingoes are in fact crossbred with wild dogs. This family run a dingo conservation park to prevent their extinction',
+ 'thumbnail': 're:http://.*\.jpg',
+ },
+ 'add_ies': ['generic'],
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+
+ release_urls_json = js_to_json(self._search_regex(
+ r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
+ webpage, ''))
+ release_urls = json.loads(release_urls_json)
+ theplatform_url = (
+ release_urls.get('progressive') or release_urls.get('standard'))
+
+ title = remove_end(self._og_search_title(webpage), ' (The Feed)')
+ description = self._html_search_meta('description', webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ return {
+ '_type': 'url_transparent',
+ 'id': video_id,
+ 'url': theplatform_url,
+
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ }
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+import base64
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ compat_urllib_request,
+ compat_urllib_parse,
+ int_or_none,
+)
+
+
+class SharedIE(InfoExtractor):
+ _VALID_URL = r'http://shared\.sx/(?P<id>[\da-z]{10})'
+
+ _TEST = {
+ 'url': 'http://shared.sx/0060718775',
+ 'md5': '106fefed92a8a2adb8c98e6a0652f49b',
+ 'info_dict': {
+ 'id': '0060718775',
+ 'ext': 'mp4',
+ 'title': 'Bmp4',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(url, video_id)
+
+ if re.search(r'>File does not exist<', page) is not None:
+ raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+ download_form = dict(re.findall(r'<input type="hidden" name="([^"]+)" value="([^"]*)"', page))
+
+ request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(download_form))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+
+ video_page = self._download_webpage(request, video_id, 'Downloading video page')
+
+ video_url = self._html_search_regex(r'data-url="([^"]+)"', video_page, 'video URL')
+ title = base64.b64decode(self._html_search_meta('full:title', page, 'title')).decode('utf-8')
+ filesize = int_or_none(self._html_search_meta('full:size', page, 'file size', fatal=False))
+ thumbnail = self._html_search_regex(
+ r'data-poster="([^"]+)"', video_page, 'thumbnail', fatal=False, default=None)
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'filesize': filesize,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ }
\ No newline at end of file
IE_DESC = 'Smotri.com community videos'
IE_NAME = 'smotri:community'
_VALID_URL = r'^https?://(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)'
+ _TEST = {
+ 'url': 'http://smotri.com/community/video/kommuna',
+ 'info_dict': {
+ 'id': 'kommuna',
+ 'title': 'КПРФ',
+ },
+ 'playlist_mincount': 4,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
IE_DESC = 'Smotri.com user videos'
IE_NAME = 'smotri:user'
_VALID_URL = r'^https?://(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)'
+ _TESTS = [{
+ 'url': 'http://smotri.com/user/inspector',
+ 'info_dict': {
+ 'id': 'inspector',
+ 'title': 'Inspector',
+ },
+ 'playlist_mincount': 9,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ float_or_none,
+ str_to_int,
+ parse_duration,
+)
+
+
+class SnotrIE(InfoExtractor):
+ _VALID_URL = r'http?://(?:www\.)?snotr\.com/video/(?P<id>\d+)/([\w]+)'
+ _TESTS = [{
+ 'url': 'http://www.snotr.com/video/13708/Drone_flying_through_fireworks',
+ 'info_dict': {
+ 'id': '13708',
+ 'ext': 'flv',
+ 'title': 'Drone flying through fireworks!',
+ 'duration': 247,
+ 'filesize_approx': 98566144,
+ 'description': 'A drone flying through Fourth of July Fireworks',
+ }
+ }, {
+ 'url': 'http://www.snotr.com/video/530/David_Letteman_-_George_W_Bush_Top_10',
+ 'info_dict': {
+ 'id': '530',
+ 'ext': 'flv',
+ 'title': 'David Letteman - George W. Bush Top 10',
+ 'duration': 126,
+ 'filesize_approx': 8912896,
+ 'description': 'The top 10 George W. Bush moments, brought to you by David Letterman!',
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ title = self._og_search_title(webpage)
+
+ description = self._og_search_description(webpage)
+ video_url = "http://cdn.videos.snotr.com/%s.flv" % video_id
+
+ view_count = str_to_int(self._html_search_regex(
+ r'<p>\n<strong>Views:</strong>\n([\d,\.]+)</p>',
+ webpage, 'view count', fatal=False))
+
+ duration = parse_duration(self._html_search_regex(
+ r'<p>\n<strong>Length:</strong>\n\s*([0-9:]+).*?</p>',
+ webpage, 'duration', fatal=False))
+
+ filesize_approx = float_or_none(self._html_search_regex(
+ r'<p>\n<strong>Filesize:</strong>\n\s*([0-9.]+)\s*megabyte</p>',
+ webpage, 'filesize', fatal=False), invscale=1024 * 1024)
+
+ return {
+ 'id': video_id,
+ 'description': description,
+ 'title': title,
+ 'url': video_url,
+ 'view_count': view_count,
+ 'duration': duration,
+ 'filesize_approx': filesize_approx,
+ }
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+from ..utils import (
+ ExtractorError,
+ compat_urllib_parse,
+ compat_urllib_request,
+ determine_ext,
+)
+import re
+
+from .common import InfoExtractor
+
+
+class SockshareIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?sockshare\.com/file/(?P<id>[0-9A-Za-z]+)'
+ _FILE_DELETED_REGEX = r'This file doesn\'t exist, or has been removed\.</div>'
+ _TEST = {
+ 'url': 'http://www.sockshare.com/file/437BE28B89D799D7',
+ 'md5': '9d0bf1cfb6dbeaa8d562f6c97506c5bd',
+ 'info_dict': {
+ 'id': '437BE28B89D799D7',
+ 'title': 'big_buck_bunny_720p_surround.avi',
+ 'ext': 'avi',
+ 'thumbnail': 're:^http://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ url = 'http://sockshare.com/file/%s' % video_id
+ webpage = self._download_webpage(url, video_id)
+
+ if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
+ raise ExtractorError('Video %s does not exist' % video_id,
+ expected=True)
+
+ confirm_hash = self._html_search_regex(r'''(?x)<input\s+
+ type="hidden"\s+
+ value="([^"]*)"\s+
+ name="hash"
+ ''', webpage, 'hash')
+
+ fields = {
+ "hash": confirm_hash,
+ "confirm": "Continue as Free User"
+ }
+
+ post = compat_urllib_parse.urlencode(fields)
+ req = compat_urllib_request.Request(url, post)
+ # Apparently, this header is required for confirmation to work.
+ req.add_header('Host', 'www.sockshare.com')
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+ webpage = self._download_webpage(
+ req, video_id, 'Downloading video page')
+
+ video_url = self._html_search_regex(
+ r'<a href="([^"]*)".+class="download_file_link"',
+ webpage, 'file url')
+ video_url = "http://www.sockshare.com" + video_url
+ title = self._html_search_regex((
+ r'<h1>(.+)<strong>',
+ r'var name = "([^"]+)";'),
+ webpage, 'title', default=None)
+ thumbnail = self._html_search_regex(
+ r'<img\s+src="([^"]*)".+?name="bg"',
+ webpage, 'thumbnail')
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': video_url,
+ 'ext': determine_ext(title),
+ }]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
_VALID_URL = r'''(?x)^(?:https?://)?
(?:(?:(?:www\.|m\.)?soundcloud\.com/
(?P<uploader>[\w\d-]+)/
- (?!sets/)(?P<title>[\w\d-]+)/?
+ (?!sets/|likes/?(?:$|[?#]))
+ (?P<title>[\w\d-]+)/?
(?P<token>[^?]+?)?(?:[?].*)?$)
|(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+))
|(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
# downloadable song
{
'url': 'https://soundcloud.com/oddsamples/bus-brakes',
- 'md5': 'fee7b8747b09bb755cefd4b853e7249a',
+ 'md5': '7624f2351f8a3b2e7cd51522496e7631',
'info_dict': {
'id': '128590877',
- 'ext': 'wav',
+ 'ext': 'mp3',
'title': 'Bus Brakes',
'description': 'md5:0170be75dd395c96025d210d261c784e',
'uploader': 'oddsamples',
class SoundcloudSetIE(SoundcloudIE):
_VALID_URL = r'https?://(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)'
IE_NAME = 'soundcloud:set'
- # it's in tests/test_playlists.py
- _TESTS = []
+ _TESTS = [{
+ 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
+ 'info_dict': {
+ 'title': 'The Royal Concept EP',
+ },
+ 'playlist_mincount': 6,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- if mobj is None:
- raise ExtractorError('Invalid URL: %s' % url)
# extract uploader (which is in the url)
uploader = mobj.group(1)
self._downloader.report_error('unable to download video webpage: %s' % compat_str(err['error_message']))
return
- self.report_extraction(full_title)
- return {'_type': 'playlist',
- 'entries': [self._extract_info_dict(track) for track in info['tracks']],
- 'id': info['id'],
- 'title': info['title'],
- }
+ return {
+ '_type': 'playlist',
+ 'entries': [self._extract_info_dict(track) for track in info['tracks']],
+ 'id': info['id'],
+ 'title': info['title'],
+ }
class SoundcloudUserIE(SoundcloudIE):
_VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
IE_NAME = 'soundcloud:user'
-
- # it's in tests/test_playlists.py
- _TESTS = []
+ _TESTS = [{
+ 'url': 'https://soundcloud.com/the-concept-band',
+ 'info_dict': {
+ 'id': '9615865',
+ 'title': 'The Royal Concept',
+ },
+ 'playlist_mincount': 12
+ }, {
+ 'url': 'https://soundcloud.com/the-concept-band/likes',
+ 'info_dict': {
+ 'id': '9615865',
+ 'title': 'The Royal Concept',
+ },
+ 'playlist_mincount': 1,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
class SoundcloudPlaylistIE(SoundcloudIE):
_VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)'
IE_NAME = 'soundcloud:playlist'
+ _TESTS = [
- # it's in tests/test_playlists.py
- _TESTS = []
+ {
+ 'url': 'http://api.soundcloud.com/playlists/4110309',
+ 'info_dict': {
+ 'id': '4110309',
+ 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
+ 'description': 're:.*?TILT Brass - Bowery Poetry Club',
+ },
+ 'playlist_count': 6,
+ }
+ ]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urllib_request,
+ parse_iso8601,
+)
+
+
+class SportDeutschlandIE(InfoExtractor):
+ _VALID_URL = r'https?://sportdeutschland\.tv/(?P<sport>[^/?#]+)/(?P<id>[^?#/]+)(?:$|[?#])'
+ _TESTS = [{
+ 'url': 'http://sportdeutschland.tv/badminton/live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
+ 'info_dict': {
+ 'id': 'live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
+ 'ext': 'mp4',
+ 'title': 'LIVE: Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen',
+ 'categories': ['Badminton'],
+ 'view_count': int,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'description': 're:^Die Badminton-WM 2014 aus Kopenhagen LIVE',
+ 'timestamp': int,
+ 'upload_date': 're:^201408[23][0-9]$',
+ },
+ 'params': {
+ 'skip_download': 'Live stream',
+ },
+ }, {
+ 'url': 'http://sportdeutschland.tv/li-ning-badminton-wm-2014/lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
+ 'info_dict': {
+ 'id': 'lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
+ 'ext': 'mp4',
+ 'upload_date': '20140825',
+ 'description': 'md5:60a20536b57cee7d9a4ec005e8687504',
+ 'timestamp': 1408976060,
+ 'title': 'Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen: Herren Einzel, Wei Lee vs. Keun Lee',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'view_count': int,
+ 'categories': ['Li-Ning Badminton WM 2014'],
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ sport_id = mobj.group('sport')
+
+ api_url = 'http://splink.tv/api/permalinks/%s/%s' % (
+ sport_id, video_id)
+ req = compat_urllib_request.Request(api_url, headers={
+ 'Accept': 'application/vnd.vidibus.v2.html+json',
+ 'Referer': url,
+ })
+ data = self._download_json(req, video_id)
+
+ categories = list(data.get('section', {}).get('tags', {}).values())
+ asset = data['asset']
+
+ formats = []
+ smil_url = asset['video']
+ if '.smil' in smil_url:
+ m3u8_url = smil_url.replace('.smil', '.m3u8')
+ formats.extend(
+ self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4'))
+
+ smil_doc = self._download_xml(
+ smil_url, video_id, note='Downloading SMIL metadata')
+ base_url = smil_doc.find('./head/meta').attrib['base']
+ formats.extend([{
+ 'format_id': 'rmtp',
+ 'url': base_url,
+ 'play_path': n.attrib['src'],
+ 'ext': 'flv',
+ 'preference': -100,
+ 'format_note': 'Seems to fail at example stream',
+ } for n in smil_doc.findall('./body/video')])
+ else:
+ formats.append({'url': smil_url})
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'formats': formats,
+ 'title': asset['title'],
+ 'thumbnail': asset.get('image'),
+ 'description': asset.get('teaser'),
+ 'categories': categories,
+ 'view_count': asset.get('views'),
+ 'rtmp_live': asset.get('live'),
+ 'timestamp': parse_iso8601(asset.get('date')),
+ }
+
'ext': 'mp4',
'upload_date': '20140329',
'title': 'FRONTIERS - Final Greenlight Trailer',
- 'description': 'md5:6df4fe8dd494ae811869672b0767e025',
+ 'description': 'md5:dc96a773669d0ca1b36c13c1f30250d9',
'uploader': 'AAD Productions',
'uploader_id': 'AtomicAgeDogGames',
}
# coding: utf-8
+from __future__ import unicode_literals
+
import re
import time
class StreamcloudIE(InfoExtractor):
- IE_NAME = u'streamcloud.eu'
+ IE_NAME = 'streamcloud.eu'
_VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)/(?P<fname>[^#?]*)\.html'
_TEST = {
- u'url': u'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
- u'file': u'skp9j99s4bpz.mp4',
- u'md5': u'6bea4c7fa5daaacc2a946b7146286686',
- u'info_dict': {
- u'title': u'youtube-dl test video \'/\\ ä ↭',
- u'duration': 9,
+ 'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
+ 'md5': '6bea4c7fa5daaacc2a946b7146286686',
+ 'info_dict': {
+ 'id': 'skp9j99s4bpz',
+ 'ext': 'mp4',
+ 'title': 'youtube-dl test video \'/\\ ä ↭',
},
- u'skip': u'Only available from the EU'
+ 'skip': 'Only available from the EU'
}
def _real_extract(self, url):
req = compat_urllib_request.Request(url, post, headers)
webpage = self._download_webpage(
- req, video_id, note=u'Downloading video page ...')
+ req, video_id, note='Downloading video page ...')
title = self._html_search_regex(
- r'<h1[^>]*>([^<]+)<', webpage, u'title')
+ r'<h1[^>]*>([^<]+)<', webpage, 'title')
video_url = self._search_regex(
- r'file:\s*"([^"]+)"', webpage, u'video URL')
- duration_str = self._search_regex(
- r'duration:\s*"?([0-9]+)"?', webpage, u'duration', fatal=False)
- duration = None if duration_str is None else int(duration_str)
+ r'file:\s*"([^"]+)"', webpage, 'video URL')
thumbnail = self._search_regex(
- r'image:\s*"([^"]+)"', webpage, u'thumbnail URL', fatal=False)
+ r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False)
return {
'id': video_id,
'title': title,
'url': video_url,
- 'duration': duration,
'thumbnail': thumbnail,
}
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ int_or_none,
+ qualities,
+ determine_ext,
+)
+
+
+class SunPornoIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?sunporno\.com/videos/(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://www.sunporno.com/videos/807778/',
+ 'md5': '6457d3c165fd6de062b99ef6c2ff4c86',
+ 'info_dict': {
+ 'id': '807778',
+ 'ext': 'flv',
+ 'title': 'md5:0a400058e8105d39e35c35e7c5184164',
+ 'description': 'md5:a31241990e1bd3a64e72ae99afb325fb',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'duration': 302,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+ description = self._html_search_meta('description', webpage, 'description')
+ thumbnail = self._html_search_regex(
+ r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
+
+ duration = parse_duration(self._search_regex(
+ r'<span>Duration: (\d+:\d+)</span>', webpage, 'duration', fatal=False))
+
+ view_count = int_or_none(self._html_search_regex(
+ r'<span class="views">(\d+)</span>', webpage, 'view count', fatal=False))
+ comment_count = int_or_none(self._html_search_regex(
+ r'(\d+)</b> Comments?', webpage, 'comment count', fatal=False))
+
+ formats = []
+ quality = qualities(['mp4', 'flv'])
+ for video_url in re.findall(r'<source src="([^"]+)"', webpage):
+ video_ext = determine_ext(video_url)
+ formats.append({
+ 'url': video_url,
+ 'format_id': video_ext,
+ 'quality': quality(video_ext),
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'comment_count': comment_count,
+ 'formats': formats,
+ 'age_limit': 18,
+ }
class SWRMediathekIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?swrmediathek\.de/player\.htm\?show=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+ _VALID_URL = r'https?://(?:www\.)?swrmediathek\.de/(?:content/)?player\.htm\?show=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
_TESTS = [{
'url': 'http://swrmediathek.de/player.htm?show=849790d0-dab8-11e3-a953-0026b975f2e6',
'uploader': 'SWR 2',
'uploader_id': '284670',
}
+ }, {
+ 'url': 'http://swrmediathek.de/content/player.htm?show=52dc7e00-15c5-11e4-84bc-0026b975f2e6',
+ 'md5': '881531487d0633080a8cc88d31ef896f',
+ 'info_dict': {
+ 'id': '52dc7e00-15c5-11e4-84bc-0026b975f2e6',
+ 'ext': 'mp4',
+ 'title': 'Familienspaß am Bodensee',
+ 'description': 'md5:0b591225a32cfde7be1629ed49fe4315',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'duration': 1784,
+ 'upload_date': '20140727',
+ 'uploader': 'SWR Fernsehen BW',
+ 'uploader_id': '281130',
+ }
}]
def _real_extract(self, url):
'description': 'md5:69da3c61275b426426d711bde96463ab',
'thumbnail': 're:^http:.*\.jpg$',
},
- }, {
- 'url': 'http://www.tagesschau.de/multimedia/video/video-5964.html',
- 'md5': '66652566900963a3f962333579eeffcf',
- 'info_dict': {
- 'id': '5964',
- 'ext': 'mp4',
- 'title': 'Nahost-Konflikt: Israel bombadiert Ziele im Gazastreifen und Westjordanland',
- 'description': 'md5:07bfc78c48eec3145ed4805299a1900a',
- 'thumbnail': 're:http://.*\.jpg',
- },
}]
_FORMATS = {
webpage = self._download_webpage(url, video_id)
- title = self._html_search_meta('title', webpage, 'title')
+ title = self._html_search_meta('title', webpage, 'title', fatal=True)
TITLE_SUFFIX = ' - TeacherTube'
if title.endswith(TITLE_SUFFIX):
title = title[:-len(TITLE_SUFFIX)].strip()
_VALID_URL = r'https?://(?:www\.)?teachertube\.com/(user/profile|collection)/(?P<user>[0-9a-zA-Z]+)/?'
- _MEDIA_RE = r'(?s)"sidebar_thumb_time">[0-9:]+</div>.+?<a href="(https?://(?:www\.)?teachertube\.com/(?:video|audio)/[^"]+)">'
+ _MEDIA_RE = r'''(?sx)
+ class="?sidebar_thumb_time"?>[0-9:]+</div>
+ \s*
+ <a\s+href="(https?://(?:www\.)?teachertube\.com/(?:video|audio)/[^"]+)"
+ '''
+ _TEST = {
+ 'url': 'http://www.teachertube.com/user/profile/rbhagwati2',
+ 'info_dict': {
+ 'id': 'rbhagwati2'
+ },
+ 'playlist_mincount': 179,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
webpage = self._download_webpage(url, user_id)
urls.extend(re.findall(self._MEDIA_RE, webpage))
- pages = re.findall(r'/ajax-user/user-videos/%s\?page=([0-9]+)' % user_id, webpage)[1:-1]
+ pages = re.findall(r'/ajax-user/user-videos/%s\?page=([0-9]+)' % user_id, webpage)[:-1]
for p in pages:
more = 'http://www.teachertube.com/ajax-user/user-videos/%s?page=%s' % (user_id, p)
- webpage = self._download_webpage(more, user_id, 'Downloading page %s/%s' % (p, len(pages) + 1))
- urls.extend(re.findall(self._MEDIA_RE, webpage))
-
- entries = []
- for url in urls:
- entries.append(self.url_result(url, 'TeacherTube'))
+ webpage = self._download_webpage(more, user_id, 'Downloading page %s/%s' % (p, len(pages)))
+ video_urls = re.findall(self._MEDIA_RE, webpage)
+ urls.extend(video_urls)
+ entries = [self.url_result(vurl, 'TeacherTube') for vurl in urls]
return self.playlist_result(entries, user_id)
video_id = mobj.group("video_id")
if not video_id:
video_id = self._html_search_regex(
- r'<article class="video" data-id="(\d+?)"',
+ r'data-node-id="(\d+?)"',
webpage, 'video id')
data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
_VALID_URL = r'https?://techtalks\.tv/talks/[^/]*/(?P<id>\d+)/'
_TEST = {
- u'url': u'http://techtalks.tv/talks/learning-topic-models-going-beyond-svd/57758/',
- u'playlist': [
+ 'url': 'http://techtalks.tv/talks/learning-topic-models-going-beyond-svd/57758/',
+ 'info_dict': {
+ 'id': '57758',
+ 'title': 'Learning Topic Models --- Going beyond SVD',
+ },
+ 'playlist': [
{
- u'file': u'57758.flv',
- u'info_dict': {
- u'title': u'Learning Topic Models --- Going beyond SVD',
+ 'info_dict': {
+ 'id': '57758',
+ 'ext': 'flv',
+ 'title': 'Learning Topic Models --- Going beyond SVD',
},
},
{
- u'file': u'57758-slides.flv',
- u'info_dict': {
- u'title': u'Learning Topic Models --- Going beyond SVD',
+ 'info_dict': {
+ 'id': '57758-slides',
+ 'ext': 'flv',
+ 'title': 'Learning Topic Models --- Going beyond SVD',
},
},
],
- u'params': {
+ 'params': {
# rtmp download
- u'skip_download': True,
+ 'skip_download': True,
},
}
mobj = re.match(self._VALID_URL, url)
talk_id = mobj.group('id')
webpage = self._download_webpage(url, talk_id)
- rtmp_url = self._search_regex(r'netConnectionUrl: \'(.*?)\'', webpage,
- u'rtmp url')
- play_path = self._search_regex(r'href=\'(.*?)\' [^>]*id="flowplayer_presenter"',
- webpage, u'presenter play path')
+ rtmp_url = self._search_regex(
+ r'netConnectionUrl: \'(.*?)\'', webpage, 'rtmp url')
+ play_path = self._search_regex(
+ r'href=\'(.*?)\' [^>]*id="flowplayer_presenter"',
+ webpage, 'presenter play path')
title = clean_html(get_element_by_attribute('class', 'title', webpage))
video_info = {
- 'id': talk_id,
- 'title': title,
- 'url': rtmp_url,
- 'play_path': play_path,
- 'ext': 'flv',
- }
+ 'id': talk_id,
+ 'title': title,
+ 'url': rtmp_url,
+ 'play_path': play_path,
+ 'ext': 'flv',
+ }
m_slides = re.search(r'<a class="slides" href=\'(.*?)\'', webpage)
if m_slides is None:
return video_info
else:
- return [
- video_info,
- # The slides video
- {
- 'id': talk_id + '-slides',
- 'title': title,
- 'url': rtmp_url,
- 'play_path': m_slides.group(1),
- 'ext': 'flv',
- },
- ]
+ return {
+ '_type': 'playlist',
+ 'id': talk_id,
+ 'title': title,
+ 'entries': [
+ video_info,
+ # The slides video
+ {
+ 'id': talk_id + '-slides',
+ 'title': title,
+ 'url': rtmp_url,
+ 'play_path': m_slides.group(1),
+ 'ext': 'flv',
+ },
+ ],
+ }
}
}, {
'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
- 'md5': '49144e345a899b8cb34d315f3b9cfeeb',
'info_dict': {
'id': '1972',
'ext': 'mp4',
'uploader': 'Gabby Giffords and Mark Kelly',
'description': 'md5:5174aed4d0f16021b704120360f72b92',
},
+ }, {
+ 'url': 'http://www.ted.com/playlists/who_are_the_hackers',
+ 'info_dict': {
+ 'id': '10',
+ 'title': 'Who are the hackers?',
+ },
+ 'playlist_mincount': 6,
}]
_NATIVE_FORMATS = {
# coding: utf-8
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ fix_xml_ampersands,
+)
+
+
+class TNAFlixIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[\w-]+)/(?P<display_id>[\w-]+)/video(?P<id>\d+)'
+
+ _TITLE_REGEX = None
+ _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
+ _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
+
+ _TEST = {
+ 'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
+ 'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+ 'info_dict': {
+ 'id': '553878',
+ 'display_id': 'Carmella-Decesare-striptease',
+ 'ext': 'mp4',
+ 'title': 'Carmella Decesare - striptease',
+ 'description': '',
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'duration': 91,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ title = self._html_search_regex(
+ self._TITLE_REGEX, webpage, 'title') if self._TITLE_REGEX else self._og_search_title(webpage)
+ description = self._html_search_regex(
+ self._DESCRIPTION_REGEX, webpage, 'description', fatal=False, default='')
+
+ age_limit = self._rta_search(webpage)
+
+ duration = self._html_search_meta('duration', webpage, 'duration', default=None)
+ if duration:
+ duration = parse_duration(duration[1:])
+
+ cfg_url = self._html_search_regex(
+ self._CONFIG_REGEX, webpage, 'flashvars.config')
+
+ cfg_xml = self._download_xml(
+ cfg_url, display_id, note='Downloading metadata',
+ transform_source=fix_xml_ampersands)
+
+ thumbnail = cfg_xml.find('./startThumb').text
+
+ formats = []
+ for item in cfg_xml.findall('./quality/item'):
+ video_url = re.sub('speed=\d+', 'speed=', item.find('videoLink').text)
+ format_id = item.find('res').text
+ fmt = {
+ 'url': video_url,
+ 'format_id': format_id,
+ }
+ m = re.search(r'^(\d+)', format_id)
+ if m:
+ fmt['height'] = int(m.group(1))
+ formats.append(fmt)
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'age_limit': age_limit,
+ 'formats': formats,
+ }
class ToypicsUserIE(InfoExtractor):
IE_DESC = 'Toypics user profile'
_VALID_URL = r'http://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
+ _TEST = {
+ 'url': 'http://videos.toypics.net/Mikey',
+ 'info_dict': {
+ 'id': 'Mikey',
+ },
+ 'playlist_mincount': 19,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
# coding: utf-8
+from __future__ import unicode_literals
+
import re
import json
class TudouIE(InfoExtractor):
_VALID_URL = r'(?:http://)?(?:www\.)?tudou\.com/(?:listplay|programs|albumplay)/(?:view|(.+?))/(?:([^/]+)|([^/]+))(?:\.html)?'
_TESTS = [{
- u'url': u'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
- u'file': u'159448201.f4v',
- u'md5': u'140a49ed444bd22f93330985d8475fcb',
- u'info_dict': {
- u"title": u"卡马乔国足开大脚长传冲吊集锦"
+ 'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
+ 'md5': '140a49ed444bd22f93330985d8475fcb',
+ 'info_dict': {
+ 'id': '159448201',
+ 'ext': 'f4v',
+ 'title': '卡马乔国足开大脚长传冲吊集锦',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }, {
+ 'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/',
+ 'info_dict': {
+ 'id': '117049447',
+ 'ext': 'f4v',
+ 'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
+ 'thumbnail': 're:^https?://.*\.jpg$',
}
- },
- {
- u'url': u'http://www.tudou.com/albumplay/TenTw_JgiPM/PzsAs5usU9A.html',
- u'file': u'todo.mp4',
- u'md5': u'todo.mp4',
- u'info_dict': {
- u'title': u'todo.mp4',
+ }, {
+ 'url': 'http://www.tudou.com/albumplay/TenTw_JgiPM/PzsAs5usU9A.html',
+ 'info_dict': {
+ 'title': 'todo.mp4',
},
- u'add_ie': [u'Youku'],
- u'skip': u'Only works from China'
+ 'add_ie': ['Youku'],
+ 'skip': 'Only works from China'
}]
def _url_for_id(self, id, quality = None):
if m and m.group(1):
return {
'_type': 'url',
- 'url': u'youku:' + m.group(1),
+ 'url': 'youku:' + m.group(1),
'ie_key': 'Youku'
}
title = self._search_regex(
- r",kw:\s*['\"](.+?)[\"']", webpage, u'title')
+ r",kw:\s*['\"](.+?)[\"']", webpage, 'title')
thumbnail_url = self._search_regex(
- r",pic:\s*[\"'](.+?)[\"']", webpage, u'thumbnail URL', fatal=False)
+ r",pic:\s*[\"'](.+?)[\"']", webpage, 'thumbnail URL', fatal=False)
segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments')
segments = json.loads(segs_json)
# It looks like the keys are the arguments that have to be passed as
# the hd field in the request url, we pick the higher
- quality = sorted(segments.keys())[-1]
+ # Also, filter non-number qualities (see issue #3643).
+ quality = sorted(filter(lambda k: k.isdigit(), segments.keys()),
+ key=lambda k: int(k))[-1]
parts = segments[quality]
result = []
len_parts = len(parts)
part_id = part['k']
final_url = self._url_for_id(part_id, quality)
ext = (final_url.split('?')[0]).split('.')[-1]
- part_info = {'id': part_id,
- 'url': final_url,
- 'ext': ext,
- 'title': title,
- 'thumbnail': thumbnail_url,
- }
+ part_info = {
+ 'id': '%s' % part_id,
+ 'url': final_url,
+ 'ext': ext,
+ 'title': title,
+ 'thumbnail': thumbnail_url,
+ }
result.append(part_info)
return result
from .common import InfoExtractor
from ..utils import (
- unified_strdate,
- clean_html,
- int_or_none,
+ float_or_none,
+ str_to_int,
)
class TvigleIE(InfoExtractor):
IE_NAME = 'tvigle'
IE_DESC = 'Интернет-телевидение Tvigle.ru'
- _VALID_URL = r'http://(?:www\.)?tvigle\.ru/category/.+?[\?&]v(?:ideo)?=(?P<id>\d+)'
+ _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$'
_TESTS = [
{
- 'url': 'http://www.tvigle.ru/category/cinema/1608/?video=503081',
- 'md5': '09afba4616666249f087efc6dcf83cb3',
+ 'url': 'http://www.tvigle.ru/video/brat-2/',
+ 'md5': '72cb7eab33e54314e1790da402d3c9c3',
'info_dict': {
- 'id': '503081',
- 'ext': 'flv',
+ 'id': '5119390',
+ 'display_id': 'brat-2',
+ 'ext': 'mp4',
'title': 'Брат 2 ',
- 'description': 'md5:f5a42970f50648cee3d7ad740f3ae769',
- 'upload_date': '20110919',
+ 'description': 'md5:5751f4fe345a58e1692585c361294bd8',
+ 'duration': 7356.369,
+ 'age_limit': 0,
},
},
{
- 'url': 'http://www.tvigle.ru/category/men/vysotskiy_vospominaniya02/?flt=196&v=676433',
- 'md5': 'e7efe5350dd5011d0de6550b53c3ba7b',
+ 'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
+ 'md5': 'd9012d7c7c598fe7a11d7fb46dc1f574',
'info_dict': {
- 'id': '676433',
- 'ext': 'flv',
+ 'id': '5142516',
+ 'ext': 'mp4',
'title': 'Ведущий телепрограммы «60 минут» (США) о Владимире Высоцком',
'description': 'md5:027f7dc872948f14c96d19b4178428a4',
- 'upload_date': '20121218',
+ 'duration': 186.080,
+ 'age_limit': 0,
},
},
]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
- video_data = self._download_xml(
- 'http://www.tvigle.ru/xml/single.php?obj=%s' % video_id, video_id, 'Downloading video XML')
+ webpage = self._download_webpage(url, display_id)
- video = video_data.find('./video')
+ video_id = self._html_search_regex(
+ r'<li class="video-preview current_playing" id="(\d+)">', webpage, 'video id')
- title = video.get('name')
- description = video.get('anons')
- if description:
- description = clean_html(description)
- thumbnail = video_data.get('img')
- upload_date = unified_strdate(video.get('date'))
- like_count = int_or_none(video.get('vtp'))
+ video_data = self._download_json(
+ 'http://cloud.tvigle.ru/api/play/video/%s/' % video_id, display_id)
- formats = []
- for num, (format_id, format_note) in enumerate([['low_file', 'SQ'], ['file', 'HQ'], ['hd', 'HD 720']]):
- video_url = video.get(format_id)
- if not video_url:
- continue
- formats.append({
- 'url': video_url,
- 'format_id': format_id,
- 'format_note': format_note,
- 'quality': num,
- })
+ item = video_data['playlist']['items'][0]
+
+ title = item['title']
+ description = item['description']
+ thumbnail = item['thumbnail']
+ duration = float_or_none(item['durationMilliseconds'], 1000)
+ age_limit = str_to_int(item['ageRestrictions'])
+ formats = []
+ for vcodec, fmts in item['videos'].items():
+ for quality, video_url in fmts.items():
+ formats.append({
+ 'url': video_url,
+ 'format_id': '%s-%s' % (vcodec, quality),
+ 'vcodec': vcodec,
+ 'height': int(quality[:-1]),
+ })
self._sort_formats(formats)
return {
'id': video_id,
+ 'display_id': display_id,
'title': title,
'description': description,
'thumbnail': thumbnail,
- 'upload_date': upload_date,
- 'like_count': like_count,
- 'age_limit': 18,
+ 'duration': duration,
+ 'age_limit': age_limit,
'formats': formats,
}
\ No newline at end of file
--- /dev/null
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ parse_iso8601,
+ qualities,
+)
+
+
+class TVPlayIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?tvplay\.lv/parraides/[^/]+/(?P<id>\d+)'
+ _TESTS = [
+ {
+ 'url': 'http://www.tvplay.lv/parraides/vinas-melo-labak/418113?autostart=true',
+ 'info_dict': {
+ 'id': '418113',
+ 'ext': 'flv',
+ 'title': 'Kādi ir īri? - Viņas melo labāk',
+ 'description': 'Baiba apsmej īrus, kādi tie ir un ko viņi dara.',
+ 'duration': 25,
+ 'timestamp': 1406097056,
+ 'upload_date': '20140723',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ video = self._download_json(
+ 'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
+
+ if video['is_geo_blocked']:
+ raise ExtractorError(
+ 'This content is not available in your country due to copyright reasons', expected=True)
+
+ streams = self._download_json(
+ 'http://playapi.mtgx.tv/v1/videos/stream/%s' % video_id, video_id, 'Downloading streams JSON')
+
+ quality = qualities(['hls', 'medium', 'high'])
+ formats = []
+ for format_id, video_url in streams['streams'].items():
+ if not video_url:
+ continue
+ fmt = {
+ 'format_id': format_id,
+ 'preference': quality(format_id),
+ }
+ if video_url.startswith('rtmp'):
+ m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url)
+ if not m:
+ continue
+ fmt.update({
+ 'ext': 'flv',
+ 'url': m.group('url'),
+ 'app': m.group('app'),
+ 'play_path': m.group('playpath'),
+ })
+ else:
+ fmt.update({
+ 'url': video_url,
+ })
+ formats.append(fmt)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': video['title'],
+ 'description': video['description'],
+ 'duration': video['duration'],
+ 'timestamp': parse_iso8601(video['created_at']),
+ 'view_count': video['views']['total'],
+ 'age_limit': video.get('age_limit', 0),
+ 'formats': formats,
+ }
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class UbuIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
+ _TEST = {
+ 'url': 'http://ubu.com/film/her_noise.html',
+ 'md5': '8edd46ee8aa6b265fb5ed6cf05c36bc9',
+ 'info_dict': {
+ 'id': 'her_noise',
+ 'ext': 'mp4',
+ 'title': 'Her Noise - The Making Of (2007)',
+ 'duration': 3600,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_regex(
+ r'<title>.+?Film & Video: ([^<]+)</title>', webpage, 'title')
+
+ duration = int_or_none(self._html_search_regex(
+ r'Duration: (\d+) minutes', webpage, 'duration', fatal=False, default=None))
+ if duration:
+ duration *= 60
+
+ formats = []
+
+ FORMAT_REGEXES = [
+ ['sq', r"'flashvars'\s*,\s*'file=([^']+)'"],
+ ['hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"']
+ ]
+
+ for format_id, format_regex in FORMAT_REGEXES:
+ m = re.search(format_regex, webpage)
+ if m:
+ formats.append({
+ 'url': m.group(1),
+ 'format_id': format_id,
+ })
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'duration': duration,
+ 'formats': formats,
+ }
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
+from ..utils import qualities
+
class UnistraIE(InfoExtractor):
- _VALID_URL = r'http://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(\d+)'
-
- _TEST = {
- u'url': u'http://utv.unistra.fr/video.php?id_video=154',
- u'file': u'154.mp4',
- u'md5': u'736f605cfdc96724d55bb543ab3ced24',
- u'info_dict': {
- u'title': u'M!ss Yella',
- u'description': u'md5:104892c71bd48e55d70b902736b81bbf',
+ _VALID_URL = r'http://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(?P<id>\d+)'
+
+ _TESTS = [
+ {
+ 'url': 'http://utv.unistra.fr/video.php?id_video=154',
+ 'md5': '736f605cfdc96724d55bb543ab3ced24',
+ 'info_dict': {
+ 'id': '154',
+ 'ext': 'mp4',
+ 'title': 'M!ss Yella',
+ 'description': 'md5:104892c71bd48e55d70b902736b81bbf',
+ },
},
- }
+ {
+ 'url': 'http://utv.unistra.fr/index.php?id_video=437',
+ 'md5': '1ddddd6cccaae76f622ce29b8779636d',
+ 'info_dict': {
+ 'id': '437',
+ 'ext': 'mp4',
+ 'title': 'Prix Louise Weiss 2014',
+ 'description': 'md5:cc3a8735f079f4fb6b0b570fc10c135a',
+ },
+ }
+ ]
def _real_extract(self, url):
- id = re.match(self._VALID_URL, url).group(1)
- webpage = self._download_webpage(url, id)
- file = re.search(r'file: "(.*?)",', webpage).group(1)
- title = self._html_search_regex(r'<title>UTV - (.*?)</', webpage, u'title')
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
- video_url = 'http://vod-flash.u-strasbg.fr:8080/' + file
+ webpage = self._download_webpage(url, video_id)
- return {'id': id,
- 'title': title,
- 'ext': 'mp4',
- 'url': video_url,
- 'description': self._html_search_regex(r'<meta name="Description" content="(.*?)"', webpage, u'description', flags=re.DOTALL),
- 'thumbnail': self._search_regex(r'image: "(.*?)"', webpage, u'thumbnail'),
- }
+ files = set(re.findall(r'file\s*:\s*"([^"]+)"', webpage))
+
+ quality = qualities(['SD', 'HD'])
+ formats = []
+ for file_path in files:
+ format_id = 'HD' if file_path.endswith('-HD.mp4') else 'SD'
+ formats.append({
+ 'url': 'http://vod-flash.u-strasbg.fr:8080%s' % file_path,
+ 'format_id': format_id,
+ 'quality': quality(format_id)
+ })
+
+ title = self._html_search_regex(
+ r'<title>UTV - (.*?)</', webpage, 'title')
+ description = self._html_search_regex(
+ r'<meta name="Description" content="(.*?)"', webpage, 'description', flags=re.DOTALL)
+ thumbnail = self._search_regex(
+ r'image: "(.*?)"', webpage, 'thumbnail')
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'formats': formats
+ }
from __future__ import unicode_literals
-import json
import re
from .common import InfoExtractor
class UstreamChannelIE(InfoExtractor):
_VALID_URL = r'https?://www\.ustream\.tv/channel/(?P<slug>.+)'
IE_NAME = 'ustream:channel'
+ _TEST = {
+ 'url': 'http://www.ustream.tv/channel/channeljapan',
+ 'info_dict': {
+ 'id': '10874166',
+ },
+ 'playlist_mincount': 54,
+ }
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
- slug = m.group('slug')
- webpage = self._download_webpage(url, slug)
+ display_id = m.group('slug')
+ webpage = self._download_webpage(url, display_id)
channel_id = get_meta_content('ustream:channel_id', webpage)
BASE = 'http://www.ustream.tv'
next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
video_ids = []
while next_url:
- reply = json.loads(self._download_webpage(compat_urlparse.urljoin(BASE, next_url), channel_id))
+ reply = self._download_json(
+ compat_urlparse.urljoin(BASE, next_url), display_id,
+ note='Downloading video information (next: %d)' % (len(video_ids) + 1))
video_ids.extend(re.findall(r'data-content-id="(\d.*)"', reply['data']))
next_url = reply['nextUrl']
- urls = ['http://www.ustream.tv/recorded/' + vid for vid in video_ids]
- url_entries = [self.url_result(eurl, 'Ustream') for eurl in urls]
- return self.playlist_result(url_entries, channel_id)
+ entries = [
+ self.url_result('http://www.ustream.tv/recorded/' + vid, 'Ustream')
+ for vid in video_ids]
+ return {
+ '_type': 'playlist',
+ 'id': channel_id,
+ 'display_id': display_id,
+ 'entries': entries,
+ }
_TEST = {
'url': 'http://veehd.com/video/4686958',
- 'file': '4686958.mp4',
'info_dict': {
+ 'id': '4686958',
+ 'ext': 'mp4',
'title': 'Time Lapse View from Space ( ISS)',
'uploader_id': 'spotted',
'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
self._downloader.report_warning(
'Cannot download SMIL information, falling back to JSON ..')
+ self._sort_formats(formats)
timestamp_ms = int(self._search_regex(
r'/Date\((\d+)\)/', video_info['launchDate'], 'launch date'))
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ float_or_none,
+ str_to_int,
+)
+
+
+class VidmeIE(InfoExtractor):
+ _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
+ _TEST = {
+ 'url': 'https://vid.me/QNB',
+ 'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
+ 'info_dict': {
+ 'id': 'QNB',
+ 'ext': 'mp4',
+ 'title': 'Fishing for piranha - the easy way',
+ 'description': 'source: https://www.facebook.com/photo.php?v=312276045600871',
+ 'duration': 119.92,
+ 'timestamp': 1406313244,
+ 'upload_date': '20140725',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(r'<source src="([^"]+)"', webpage, 'video URL')
+
+ title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage, default='')
+ thumbnail = self._og_search_thumbnail(webpage)
+ timestamp = int_or_none(self._og_search_property('updated_time', webpage, fatal=False))
+ width = int_or_none(self._og_search_property('video:width', webpage, fatal=False))
+ height = int_or_none(self._og_search_property('video:height', webpage, fatal=False))
+ duration = float_or_none(self._html_search_regex(
+ r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
+ view_count = str_to_int(self._html_search_regex(
+ r'<span class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
+ like_count = str_to_int(self._html_search_regex(
+ r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
+ webpage, 'like count', fatal=False))
+ comment_count = str_to_int(self._html_search_regex(
+ r'class="js-comment-count"[^>]+data-count="([\d,\.\s]+)">',
+ webpage, 'comment count', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'timestamp': timestamp,
+ 'width': width,
+ 'height': height,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'like_count': like_count,
+ 'comment_count': comment_count,
+ }
(?P<proto>(?:https?:)?//)?
(?:(?:www|(?P<player>player))\.)?
vimeo(?P<pro>pro)?\.com/
+ (?!channels/[^/?#]+/?(?:$|[?#])|album/)
(?:.*?/)?
(?:(?:play_redirect_hls|moogaloop\.swf)\?clip_id=)?
(?:videos?/)?
'info_dict': {
'id': '54469442',
'ext': 'mp4',
- 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software',
+ 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software 2012',
'uploader': 'The BLN & Business of Software',
'uploader_id': 'theblnbusinessofsoftware',
'duration': 3610,
'videopassword': 'youtube-dl',
},
},
+ {
+ 'url': 'http://vimeo.com/channels/keypeele/75629013',
+ 'md5': '2f86a05afe9d7abc0b9126d229bbe15d',
+ 'note': 'Video is freely available via original URL '
+ 'and protected with password when accessed via http://vimeo.com/75629013',
+ 'info_dict': {
+ 'id': '75629013',
+ 'ext': 'mp4',
+ 'title': 'Key & Peele: Terrorist Interrogation',
+ 'description': 'md5:8678b246399b070816b12313e8b4eb5c',
+ 'uploader_id': 'atencio',
+ 'uploader': 'Peter Atencio',
+ 'duration': 187,
+ },
+ },
{
'url': 'http://vimeo.com/76979871',
'md5': '3363dd6ffebe3784d56f4132317fd446',
},
]
- @classmethod
- def suitable(cls, url):
- if VimeoChannelIE.suitable(url):
- # Otherwise channel urls like http://vimeo.com/channels/31259 would
- # match
- return False
- else:
- return super(VimeoIE, cls).suitable(url)
-
def _verify_video_password(self, url, video_id, webpage):
password = self._downloader.params.get('videopassword', None)
if password is None:
if data is not None:
headers = headers.copy()
headers.update(data)
+ if 'Referer' not in headers:
+ headers['Referer'] = url
# Extract ID from URL
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
if mobj.group('pro') or mobj.group('player'):
url = 'http://player.vimeo.com/video/' + video_id
- else:
- url = 'https://vimeo.com/' + video_id
# Retrieve video webpage to extract further information
request = compat_urllib_request.Request(url, None, headers)
if video_thumbnail is None:
video_thumbs = config["video"].get("thumbs")
if video_thumbs and isinstance(video_thumbs, dict):
- _, video_thumbnail = sorted((int(width), t_url) for (width, t_url) in video_thumbs.items())[-1]
+ _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
# Extract video description
video_description = None
class VimeoChannelIE(InfoExtractor):
IE_NAME = 'vimeo:channel'
- _VALID_URL = r'(?:https?://)?vimeo\.com/channels/(?P<id>[^/]+)/?(\?.*)?$'
+ _VALID_URL = r'https?://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
_MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
_TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
+ _TESTS = [{
+ 'url': 'http://vimeo.com/channels/tributes',
+ 'info_dict': {
+ 'title': 'Vimeo Tributes',
+ },
+ 'playlist_mincount': 25,
+ }]
def _page_url(self, base_url, pagenum):
return '%s/videos/page:%d/' % (base_url, pagenum)
class VimeoUserIE(VimeoChannelIE):
IE_NAME = 'vimeo:user'
- _VALID_URL = r'(?:https?://)?vimeo\.com/(?P<name>[^/]+)(?:/videos|[#?]|$)'
+ _VALID_URL = r'https?://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
_TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
-
- @classmethod
- def suitable(cls, url):
- if VimeoChannelIE.suitable(url) or VimeoIE.suitable(url) or VimeoAlbumIE.suitable(url) or VimeoGroupsIE.suitable(url):
- return False
- return super(VimeoUserIE, cls).suitable(url)
+ _TESTS = [{
+ 'url': 'http://vimeo.com/nkistudio/videos',
+ 'info_dict': {
+ 'title': 'Nki',
+ },
+ 'playlist_mincount': 66,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
class VimeoAlbumIE(VimeoChannelIE):
IE_NAME = 'vimeo:album'
- _VALID_URL = r'(?:https?://)?vimeo\.com/album/(?P<id>\d+)'
+ _VALID_URL = r'https?://vimeo\.com/album/(?P<id>\d+)'
_TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
+ _TESTS = [{
+ 'url': 'http://vimeo.com/album/2632481',
+ 'info_dict': {
+ 'title': 'Staff Favorites: November 2013',
+ },
+ 'playlist_mincount': 13,
+ }]
def _page_url(self, base_url, pagenum):
return '%s/page:%d/' % (base_url, pagenum)
class VimeoGroupsIE(VimeoAlbumIE):
IE_NAME = 'vimeo:group'
_VALID_URL = r'(?:https?://)?vimeo\.com/groups/(?P<name>[^/]+)'
+ _TESTS = [{
+ 'url': 'http://vimeo.com/groups/rolexawards',
+ 'info_dict': {
+ 'title': 'Rolex Awards for Enterprise',
+ },
+ 'playlist_mincount': 73,
+ }]
def _extract_list_title(self, webpage):
return self._og_search_title(webpage)
class VimeoReviewIE(InfoExtractor):
IE_NAME = 'vimeo:review'
IE_DESC = 'Review pages on vimeo'
- _VALID_URL = r'(?:https?://)?vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
- _TEST = {
+ _VALID_URL = r'https?://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
+ _TESTS = [{
'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
'file': '75524534.mp4',
'md5': 'c507a72f780cacc12b2248bb4006d253',
'title': "DICK HARDWICK 'Comedian'",
'uploader': 'Richard Hardwick',
}
- }
+ }, {
+ 'note': 'video player needs Referer',
+ 'url': 'http://vimeo.com/user22258446/review/91613211/13f927e053',
+ 'md5': '6295fdab8f4bf6a002d058b2c6dce276',
+ 'info_dict': {
+ 'id': '91613211',
+ 'ext': 'mp4',
+ 'title': 'Death by dogma versus assembling agile - Sander Hoogendoorn',
+ 'uploader': 'DevWeek Events',
+ 'duration': 2773,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
_VALID_URL = r'https?://vimeo\.com/home/watchlater|:vimeowatchlater'
_LOGIN_REQUIRED = True
_TITLE_RE = r'href="/home/watchlater".*?>(.*?)<'
+ _TESTS = [{
+ 'url': 'http://vimeo.com/home/watchlater',
+ 'only_matching': True,
+ }]
def _real_initialize(self):
self._login()
IE_NAME = 'vine:user'
_VALID_URL = r'(?:https?://)?vine\.co/(?P<user>[^/]+)/?(\?.*)?$'
_VINE_BASE_URL = "https://vine.co/"
+ _TEST = {
+ 'url': 'https://vine.co/Visa',
+ 'info_dict': {
+ 'id': 'Visa',
+ },
+ 'playlist_mincount': 47,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
class VodlockerIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?vodlocker.com/(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
+ _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
_TESTS = [{
'url': 'http://vodlocker.com/e8wvyzz4sl42',
req, video_id, 'Downloading video page')
title = self._search_regex(
- r'id="file_title".*?>\s*(.*?)\s*<span', webpage, 'title')
+ r'id="file_title".*?>\s*(.*?)\s*<(?:br|span)', webpage, 'title')
thumbnail = self._search_regex(
r'image:\s*"(http[^\"]+)",', webpage, 'thumbnail')
url = self._search_regex(
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ str_to_int,
+)
+
+
+class VpornIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?vporn\.com/[^/]+/(?P<display_id>[^/]+)/(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://www.vporn.com/masturbation/violet-on-her-th-birthday/497944/',
+ 'md5': 'facf37c1b86546fa0208058546842c55',
+ 'info_dict': {
+ 'id': '497944',
+ 'display_id': 'violet-on-her-th-birthday',
+ 'ext': 'mp4',
+ 'title': 'Violet on her 19th birthday',
+ 'description': 'Violet dances in front of the camera which is sure to get you horny.',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'uploader': 'kileyGrope',
+ 'categories': ['Masturbation', 'Teen'],
+ 'duration': 393,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ title = self._html_search_regex(
+ r'videoname\s*=\s*\'([^\']+)\'', webpage, 'title').strip()
+ description = self._html_search_regex(
+ r'<div class="description_txt">(.*?)</div>', webpage, 'description', fatal=False)
+ thumbnail = self._html_search_regex(
+ r'flashvars\.imageUrl\s*=\s*"([^"]+)"', webpage, 'description', fatal=False, default=None)
+ if thumbnail:
+ thumbnail = 'http://www.vporn.com' + thumbnail
+
+ uploader = self._html_search_regex(
+ r'(?s)UPLOADED BY.*?<a href="/user/[^"]+">([^<]+)</a>',
+ webpage, 'uploader', fatal=False)
+
+ categories = re.findall(r'<a href="/cat/[^"]+">([^<]+)</a>', webpage)
+
+ duration = parse_duration(self._search_regex(
+ r'duration (\d+ min \d+ sec)', webpage, 'duration', fatal=False))
+
+ view_count = str_to_int(self._html_search_regex(
+ r'<span>([\d,\.]+) VIEWS</span>', webpage, 'view count', fatal=False))
+ like_count = str_to_int(self._html_search_regex(
+ r'<span id="like" class="n">([\d,\.]+)</span>', webpage, 'like count', fatal=False))
+ dislike_count = str_to_int(self._html_search_regex(
+ r'<span id="dislike" class="n">([\d,\.]+)</span>', webpage, 'dislike count', fatal=False))
+ comment_count = str_to_int(self._html_search_regex(
+ r'<h4>Comments \(<b>([\d,\.]+)</b>\)</h4>', webpage, 'comment count', fatal=False))
+
+ formats = []
+
+ for video in re.findall(r'flashvars\.videoUrl([^=]+?)\s*=\s*"([^"]+)"', webpage):
+ video_url = video[1]
+ fmt = {
+ 'url': video_url,
+ 'format_id': video[0],
+ }
+ m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)_(?P<vbr>\d+)k\.mp4$', video_url)
+ if m:
+ fmt.update({
+ 'width': int(m.group('width')),
+ 'height': int(m.group('height')),
+ 'vbr': int(m.group('vbr')),
+ })
+ formats.append(fmt)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'categories': categories,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'like_count': like_count,
+ 'dislike_count': dislike_count,
+ 'comment_count': comment_count,
+ 'age_limit': 18,
+ 'formats': formats,
+ }
import re
from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+ int_or_none,
+ compat_str,
+)
class VubeIE(InfoExtractor):
'ext': 'mp4',
'title': 'Chiara Grispo - Price Tag by Jessie J',
'description': 'md5:8ea652a1f36818352428cb5134933313',
- 'thumbnail': 'http://frame.thestaticvube.com/snap/228x128/102e7e63057-5ebc-4f5c-4065-6ce4ebde131f.jpg',
+ 'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/102e7e63057-5ebc-4f5c-4065-6ce4ebde131f\.jpg$',
'uploader': 'Chiara.Grispo',
- 'uploader_id': '1u3hX0znhP',
'timestamp': 1388743358,
'upload_date': '20140103',
- 'duration': 170.56
+ 'duration': 170.56,
+ 'like_count': int,
+ 'dislike_count': int,
+ 'comment_count': int,
+ 'categories': ['pop', 'music', 'cover', 'singing', 'jessie j', 'price tag', 'chiara grispo'],
}
},
{
'ext': 'mp4',
'title': 'My 7 year old Sister and I singing "Alive" by Krewella',
'description': 'md5:40bcacb97796339f1690642c21d56f4a',
- 'thumbnail': 'http://frame.thestaticvube.com/snap/228x128/102265d5a9f-0f17-4f6b-5753-adf08484ee1e.jpg',
+ 'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/102265d5a9f-0f17-4f6b-5753-adf08484ee1e\.jpg$',
'uploader': 'Seraina',
- 'uploader_id': 'XU9VE2BQ2q',
'timestamp': 1396492438,
'upload_date': '20140403',
- 'duration': 240.107
+ 'duration': 240.107,
+ 'like_count': int,
+ 'dislike_count': int,
+ 'comment_count': int,
+ 'categories': ['seraina', 'jessica', 'krewella', 'alive'],
+ }
+ }, {
+ 'url': 'http://vube.com/vote/Siren+Gene/0nmsMY5vEq?n=2&t=s',
+ 'md5': '0584fc13b50f887127d9d1007589d27f',
+ 'info_dict': {
+ 'id': '0nmsMY5vEq',
+ 'ext': 'mp4',
+ 'title': 'Frozen - Let It Go Cover by Siren Gene',
+ 'description': 'My rendition of "Let It Go" originally sung by Idina Menzel.',
+ 'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/10283ab622a-86c9-4681-51f2-30d1f65774af\.jpg$',
+ 'uploader': 'Siren',
+ 'timestamp': 1395448018,
+ 'upload_date': '20140322',
+ 'duration': 221.788,
+ 'like_count': int,
+ 'dislike_count': int,
+ 'comment_count': int,
+ 'categories': ['let it go', 'cover', 'idina menzel', 'frozen', 'singing', 'disney', 'siren gene'],
}
}
]
video_id = mobj.group('id')
video = self._download_json(
- 'http://vube.com/api/v2/video/%s' % video_id, video_id, 'Downloading video JSON')
+ 'http://vube.com/t-api/v1/video/%s' % video_id, video_id, 'Downloading video JSON')
public_id = video['public_id']
- formats = [
- {
- 'url': 'http://video.thestaticvube.com/video/%s/%s.mp4' % (fmt['media_resolution_id'], public_id),
- 'height': int(fmt['height']),
- 'abr': int(fmt['audio_bitrate']),
- 'vbr': int(fmt['video_bitrate']),
- 'format_id': fmt['media_resolution_id']
- } for fmt in video['mtm'] if fmt['transcoding_status'] == 'processed'
- ]
+ formats = []
+
+ for media in video['media'].get('video', []) + video['media'].get('audio', []):
+ if media['transcoding_status'] != 'processed':
+ continue
+ fmt = {
+ 'url': 'http://video.thestaticvube.com/video/%s/%s.mp4' % (media['media_resolution_id'], public_id),
+ 'abr': int(media['audio_bitrate']),
+ 'format_id': compat_str(media['media_resolution_id']),
+ }
+ vbr = int(media['video_bitrate'])
+ if vbr:
+ fmt.update({
+ 'vbr': vbr,
+ 'height': int(media['height']),
+ })
+ formats.append(fmt)
self._sort_formats(formats)
title = video['title']
description = video.get('description')
- thumbnail = video['thumbnail_src']
- if thumbnail.startswith('//'):
- thumbnail = 'http:' + thumbnail
- uploader = video['user_alias']
- uploader_id = video['user_url_id']
- timestamp = int(video['upload_time'])
+ thumbnail = self._proto_relative_url(video.get('thumbnail_src'), scheme='http:')
+ uploader = video.get('user_alias') or video.get('channel')
+ timestamp = int_or_none(video.get('upload_time'))
duration = video['duration']
view_count = video.get('raw_view_count')
like_count = video.get('total_likes')
- dislike_count= video.get('total_hates')
+ dislike_count = video.get('total_hates')
- comment = self._download_json(
- 'http://vube.com/api/video/%s/comment' % video_id, video_id, 'Downloading video comment JSON')
+ comments = video.get('comments')
+ comment_count = None
+ if comments is None:
+ comment_data = self._download_json(
+ 'http://vube.com/api/video/%s/comment' % video_id,
+ video_id, 'Downloading video comment JSON', fatal=False)
+ if comment_data is not None:
+ comment_count = int_or_none(comment_data.get('total'))
+ else:
+ comment_count = len(comments)
- comment_count = int_or_none(comment.get('total'))
+ categories = [tag['text'] for tag in video['tags']]
return {
'id': video_id,
'description': description,
'thumbnail': thumbnail,
'uploader': uploader,
- 'uploader_id': uploader_id,
'timestamp': timestamp,
'duration': duration,
'view_count': view_count,
'like_count': like_count,
'dislike_count': dislike_count,
'comment_count': comment_count,
+ 'categories': categories,
}
_VALID_URL = r'^https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
_TEST = {
'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
+ 'info_dict': {
+ 'title': 'Sinkhole of bureaucracy',
+ },
'playlist': [{
'md5': 'c3f4b4922ffa259243f68e928db2db8c',
'info_dict': {
from __future__ import unicode_literals
import re
+import hashlib
from .common import InfoExtractor
-from ..utils import (
- unified_strdate,
-)
+from ..utils import unified_strdate
class WatIE(InfoExtractor):
- _VALID_URL = r'http://www\.wat\.tv/.*-(?P<shortID>.*?)_.*?\.html'
+ _VALID_URL = r'http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html'
IE_NAME = 'wat.tv'
- _TEST = {
- 'url': 'http://www.wat.tv/video/world-war-philadelphia-vost-6bv55_2fjr7_.html',
- 'info_dict': {
- 'id': '10631273',
- 'ext': 'mp4',
- 'title': 'World War Z - Philadelphia VOST',
- 'description': 'La menace est partout. Que se passe-t-il à Philadelphia ?\r\nWORLD WAR Z, avec Brad Pitt, au cinéma le 3 juillet.\r\nhttp://www.worldwarz.fr',
+ _TESTS = [
+ {
+ 'url': 'http://www.wat.tv/video/soupe-figues-l-orange-aux-epices-6z1uz_2hvf7_.html',
+ 'md5': 'ce70e9223945ed26a8056d413ca55dc9',
+ 'info_dict': {
+ 'id': '11713067',
+ 'display_id': 'soupe-figues-l-orange-aux-epices',
+ 'ext': 'mp4',
+ 'title': 'Soupe de figues à l\'orange et aux épices',
+ 'description': 'Retrouvez l\'émission "Petits plats en équilibre", diffusée le 18 août 2014.',
+ 'upload_date': '20140819',
+ 'duration': 120,
+ },
},
- 'params': {
- # Sometimes wat serves the whole file with the --test option
- 'skip_download': True,
+ {
+ 'url': 'http://www.wat.tv/video/gregory-lemarchal-voix-ange-6z1v7_6ygkj_.html',
+ 'md5': 'fbc84e4378165278e743956d9c1bf16b',
+ 'info_dict': {
+ 'id': '11713075',
+ 'display_id': 'gregory-lemarchal-voix-ange',
+ 'ext': 'mp4',
+ 'title': 'Grégory Lemarchal, une voix d\'ange depuis 10 ans (1/3)',
+ 'description': 'md5:b7a849cf16a2b733d9cd10c52906dee3',
+ 'upload_date': '20140816',
+ 'duration': 2910,
+ },
},
- }
+ ]
def download_video_info(self, real_id):
# 'contentv4' is used in the website, but it also returns the related
def real_id_for_chapter(chapter):
return chapter['tc_start'].split('-')[0]
mobj = re.match(self._VALID_URL, url)
- short_id = mobj.group('shortID')
- webpage = self._download_webpage(url, short_id)
+ short_id = mobj.group('short_id')
+ display_id = mobj.group('display_id')
+ webpage = self._download_webpage(url, display_id or short_id)
real_id = self._search_regex(r'xtpage = ".*-(.*?)";', webpage, 'real id')
video_info = self.download_video_info(real_id)
+
+ geo_list = video_info.get('geoList')
+ country = geo_list[0] if geo_list else ''
+
chapters = video_info['chapters']
first_chapter = chapters[0]
+ files = video_info['files']
+ first_file = files[0]
if real_id_for_chapter(first_chapter) != real_id:
self.to_screen('Multipart video detected')
upload_date = unified_strdate(first_chapter['date_diffusion'])
# Otherwise we can continue and extract just one part, we have to use
# the short id for getting the video url
+
+ formats = [{
+ 'url': 'http://wat.tv/get/android5/%s.mp4' % real_id,
+ 'format_id': 'Mobile',
+ }]
+
+ fmts = [('SD', 'web')]
+ if first_file.get('hasHD'):
+ fmts.append(('HD', 'webhd'))
+
+ def compute_token(param):
+ timestamp = '%08x' % int(self._download_webpage(
+ 'http://www.wat.tv/servertime', real_id,
+ 'Downloading server time').split('|')[0])
+ magic = '9b673b13fa4682ed14c3cfa5af5310274b514c4133e9b3a81e6e3aba009l2564'
+ return '%s/%s' % (hashlib.md5((magic + param + timestamp).encode('ascii')).hexdigest(), timestamp)
+
+ for fmt in fmts:
+ webid = '/%s/%s' % (fmt[1], real_id)
+ video_url = self._download_webpage(
+ 'http://www.wat.tv/get%s?token=%s&getURL=1&country=%s' % (webid, compute_token(webid), country),
+ real_id,
+ 'Downloding %s video URL' % fmt[0],
+ 'Failed to download %s video URL' % fmt[0],
+ False)
+ if not video_url:
+ continue
+ formats.append({
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'format_id': fmt[0],
+ })
+
return {
'id': real_id,
- 'url': 'http://wat.tv/get/android5/%s.mp4' % real_id,
+ 'display_id': display_id,
'title': first_chapter['title'],
'thumbnail': first_chapter['preview'],
'description': first_chapter['description'],
'view_count': video_info['views'],
'upload_date': upload_date,
+ 'duration': first_file['duration'],
+ 'formats': formats,
}
--- /dev/null
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class WayOfTheMasterIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.wayofthemaster\.com/([^/?#]*/)*(?P<id>[^/?#]+)\.s?html(?:$|[?#])'
+
+ _TEST = {
+ 'url': 'http://www.wayofthemaster.com/hbks.shtml',
+ 'md5': '5316b57487ada8480606a93cb3d18d24',
+ 'info_dict': {
+ 'id': 'hbks',
+ 'ext': 'mp4',
+ 'title': 'Intelligent Design vs. Evolution',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._search_regex(
+ r'<img src="images/title_[^"]+".*?alt="([^"]+)"',
+ webpage, 'title', default=None)
+ if title is None:
+ title = self._html_search_regex(
+ r'<title>(.*?)</title>', webpage, 'page title')
+
+ url_base = self._search_regex(
+ r'<param\s+name="?movie"?\s+value=".*?/wotm_videoplayer_highlow[0-9]*\.swf\?vid=([^"]+)"',
+ webpage, 'URL base')
+ formats = [{
+ 'format_id': 'low',
+ 'quality': 1,
+ 'url': url_base + '_low.mp4',
+ }, {
+ 'format_id': 'high',
+ 'quality': 2,
+ 'url': url_base + '_high.mp4',
+ }]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ }
},
},
{
- 'url': 'http://www.funkhauseuropa.de/av/audiosuepersongsoulbossanova100-audioplayer.html',
- 'md5': '24e83813e832badb0a8d7d1ef9ef0691',
+ 'url': 'http://www.funkhauseuropa.de/av/audioflaviacoelhoamaramar100-audioplayer.html',
+ 'md5': '99a1443ff29af19f6c52cf6f4dc1f4aa',
'info_dict': {
- 'id': 'mdb-463528',
+ 'id': 'mdb-478135',
'ext': 'mp3',
- 'title': 'Süpersong: Soul Bossa Nova',
+ 'title': 'Flavia Coelho: Amar é Amar',
'description': 'md5:7b29e97e10dfb6e265238b32fa35b23a',
- 'upload_date': '20140630',
+ 'upload_date': '20140717',
},
},
]
]
return self.playlist_result(entries, page_id)
- flashvars = compat_urlparse.parse_qs(
+ flashvars = compat_parse_qs(
self._html_search_regex(r'<param name="flashvars" value="([^"]+)"', webpage, 'flashvars'))
page_id = flashvars['trackerClipId'][0]
--- /dev/null
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_iso8601,
+ float_or_none,
+ int_or_none,
+)
+
+
+class XboxClipsIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/video\.php\?.*vid=(?P<id>[\w-]{36})'
+ _TEST = {
+ 'url': 'https://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325',
+ 'md5': 'fbe1ec805e920aeb8eced3c3e657df5d',
+ 'info_dict': {
+ 'id': '074a69a9-5faf-46aa-b93b-9909c1720325',
+ 'ext': 'mp4',
+ 'title': 'Iabdulelah playing Upload Studio',
+ 'filesize_approx': 28101836.8,
+ 'timestamp': 1407388500,
+ 'upload_date': '20140807',
+ 'duration': 56,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(
+ r'>Link: <a href="([^"]+)">', webpage, 'video URL')
+ title = self._html_search_regex(
+ r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
+ timestamp = parse_iso8601(self._html_search_regex(
+ r'>Recorded: ([^<]+)<', webpage, 'upload date', fatal=False))
+ filesize = float_or_none(self._html_search_regex(
+ r'>Size: ([\d\.]+)MB<', webpage, 'file size', fatal=False), invscale=1024 * 1024)
+ duration = int_or_none(self._html_search_regex(
+ r'>Duration: (\d+) Seconds<', webpage, 'duration', fatal=False))
+ view_count = int_or_none(self._html_search_regex(
+ r'>Views: (\d+)<', webpage, 'view count', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'timestamp': timestamp,
+ 'filesize_approx': filesize,
+ 'duration': duration,
+ 'view_count': view_count,
+ }
class XHamsterIE(InfoExtractor):
"""Information Extractor for xHamster"""
- _VALID_URL = r'http://(?:www\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
+ _VALID_URL = r'http://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
_TESTS = [
{
'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
'age_limit': 18,
}
+
class XTubeUserIE(InfoExtractor):
IE_DESC = 'XTube user profile'
_VALID_URL = r'https?://(?:www\.)?xtube\.com/community/profile\.php\?(.*?)user=(?P<username>[^&#]+)(?:$|[&#])'
+ _TEST = {
+ 'url': 'http://www.xtube.com/community/profile.php?user=greenshowers',
+ 'info_dict': {
+ 'id': 'greenshowers',
+ },
+ 'playlist_mincount': 155,
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
class YahooIE(InfoExtractor):
IE_DESC = 'Yahoo screen and movies'
- _VALID_URL = r'https?://(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html'
+ _VALID_URL = r'(?P<url>https?://(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
_TESTS = [
{
'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
'title': 'The World Loves Spider-Man',
'description': '''People all over the world are celebrating the release of \"The Amazing Spider-Man 2.\" We're taking a look at the enthusiastic response Spider-Man has received from viewers all over the world.''',
}
- }
+ },
+ {
+ 'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
+ 'md5': '60e8ac193d8fb71997caa8fce54c6460',
+ 'info_dict': {
+ 'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
+ 'ext': 'mp4',
+ 'title': "Yahoo Saves 'Community'",
+ 'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
+ }
+ },
]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
+ url = mobj.group('url')
webpage = self._download_webpage(url, video_id)
items_json = self._search_regex(
if items_json is None:
CONTENT_ID_REGEXES = [
r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
- r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"'
+ r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
+ r'"first_videoid"\s*:\s*"([^"]+)"',
]
long_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
video_id = long_id
# coding: utf-8
-import collections
-import errno
-import io
import itertools
import json
import os.path
import re
-import struct
import traceback
-import zlib
from .common import InfoExtractor, SearchInfoExtractor
from .subtitles import SubtitlesInfoExtractor
from ..jsinterp import JSInterpreter
+from ..swfinterp import SWFInterpreter
from ..utils import (
compat_chr,
compat_parse_qs,
compat_str,
clean_html,
- get_cachedir,
get_element_by_id,
get_element_by_attribute,
ExtractorError,
unescapeHTML,
unified_strdate,
orderedSet,
- write_json_file,
uppercase_escape,
)
class YoutubeBaseInfoExtractor(InfoExtractor):
"""Provide base functions for Youtube extractors"""
_LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
+ _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor'
_LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
_AGE_URL = 'https://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
_NETRC_MACHINE = 'youtube'
fatal=False))
def _login(self):
+ """
+ Attempt to log in to YouTube.
+ True is returned if successful or skipped.
+ False is returned if login failed.
+
+ If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
+ """
(username, password) = self._get_login_info()
# No authentication to be performed
if username is None:
if self._LOGIN_REQUIRED:
raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
- return False
+ return True
login_page = self._download_webpage(
self._LOGIN_URL, None,
u'Email': username,
u'GALX': galx,
u'Passwd': password,
+
u'PersistentCookie': u'yes',
u'_utf8': u'霱',
u'bgresponse': u'js_disabled',
u'uilel': u'3',
u'hl': u'en_US',
}
+
# Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
# chokes on unicode
login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
note=u'Logging in', errnote=u'unable to log in', fatal=False)
if login_results is False:
return False
+
+ if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
+ raise ExtractorError(u'Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
+
+ # Two-Factor
+ # TODO add SMS and phone call support - these require making a request and then prompting the user
+
+ if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', login_results) is not None:
+ tfa_code = self._get_tfa_info()
+
+ if tfa_code is None:
+ self._downloader.report_warning(u'Two-factor authentication required. Provide it with --twofactor <code>')
+ self._downloader.report_warning(u'(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+ return False
+
+ # Unlike the first login form, secTok and timeStmp are both required for the TFA form
+
+ match = re.search(r'id="secTok"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
+ if match is None:
+ self._downloader.report_warning(u'Failed to get secTok - did the page structure change?')
+ secTok = match.group(1)
+ match = re.search(r'id="timeStmp"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
+ if match is None:
+ self._downloader.report_warning(u'Failed to get timeStmp - did the page structure change?')
+ timeStmp = match.group(1)
+
+ tfa_form_strs = {
+ u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
+ u'smsToken': u'',
+ u'smsUserPin': tfa_code,
+ u'smsVerifyPin': u'Verify',
+
+ u'PersistentCookie': u'yes',
+ u'checkConnection': u'',
+ u'checkedDomains': u'youtube',
+ u'pstMsg': u'1',
+ u'secTok': secTok,
+ u'timeStmp': timeStmp,
+ u'service': u'youtube',
+ u'hl': u'en_US',
+ }
+ tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in tfa_form_strs.items())
+ tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
+
+ tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
+ tfa_results = self._download_webpage(
+ tfa_req, None,
+ note=u'Submitting TFA code', errnote=u'unable to submit tfa', fatal=False)
+
+ if tfa_results is False:
+ return False
+
+ if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', tfa_results) is not None:
+ self._downloader.report_warning(u'Two-factor code expired. Please try again, or use a one-use backup code instead.')
+ return False
+ if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
+ self._downloader.report_warning(u'unable to log in - did the page structure change?')
+ return False
+ if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
+ self._downloader.report_warning(u'Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
+ return False
+
if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
self._downloader.report_warning(u'unable to log in: bad username or password')
return False
'272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
# Dash webm audio
- '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 48, 'preference': -50},
+ '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
'172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
# RTMP (unnamed)
u"upload_date": u"20121002",
u"description": u"test chars: \"'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .",
u"categories": [u'Science & Technology'],
+ 'like_count': int,
+ 'dislike_count': int,
}
},
{
"""Indicate the download will use the RTMP protocol."""
self.to_screen(u'RTMP download detected')
- def _extract_signature_function(self, video_id, player_url, slen):
- id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
- player_url)
+ def _signature_cache_id(self, example_sig):
+ """ Return a string representation of a signature """
+ return u'.'.join(compat_str(len(part)) for part in example_sig.split('.'))
+
+ def _extract_signature_function(self, video_id, player_url, example_sig):
+ id_m = re.match(
+ r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
+ player_url)
+ if not id_m:
+ raise ExtractorError('Cannot identify player %r' % player_url)
player_type = id_m.group('ext')
player_id = id_m.group('id')
# Read from filesystem cache
- func_id = '%s_%s_%d' % (player_type, player_id, slen)
+ func_id = '%s_%s_%s' % (
+ player_type, player_id, self._signature_cache_id(example_sig))
assert os.path.basename(func_id) == func_id
- cache_dir = get_cachedir(self._downloader.params)
- cache_enabled = cache_dir is not None
- if cache_enabled:
- cache_fn = os.path.join(os.path.expanduser(cache_dir),
- u'youtube-sigfuncs',
- func_id + '.json')
- try:
- with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
- cache_spec = json.load(cachef)
- return lambda s: u''.join(s[i] for i in cache_spec)
- except IOError:
- pass # No cache available
+ cache_spec = self._downloader.cache.load(u'youtube-sigfuncs', func_id)
+ if cache_spec is not None:
+ return lambda s: u''.join(s[i] for i in cache_spec)
if player_type == 'js':
code = self._download_webpage(
else:
assert False, 'Invalid player type %r' % player_type
- if cache_enabled:
- try:
- test_string = u''.join(map(compat_chr, range(slen)))
- cache_res = res(test_string)
- cache_spec = [ord(c) for c in cache_res]
- try:
- os.makedirs(os.path.dirname(cache_fn))
- except OSError as ose:
- if ose.errno != errno.EEXIST:
- raise
- write_json_file(cache_spec, cache_fn)
- except Exception:
- tb = traceback.format_exc()
- self._downloader.report_warning(
- u'Writing cache to %r failed: %s' % (cache_fn, tb))
+ if cache_spec is None:
+ test_string = u''.join(map(compat_chr, range(len(example_sig))))
+ cache_res = res(test_string)
+ cache_spec = [ord(c) for c in cache_res]
+ self._downloader.cache.store(u'youtube-sigfuncs', func_id, cache_spec)
return res
- def _print_sig_code(self, func, slen):
+ def _print_sig_code(self, func, example_sig):
def gen_sig_code(idxs):
def _genslice(start, end, step):
starts = u'' if start == 0 else str(start)
else:
yield _genslice(start, i, step)
- test_string = u''.join(map(compat_chr, range(slen)))
+ test_string = u''.join(map(compat_chr, range(len(example_sig))))
cache_res = func(test_string)
cache_spec = [ord(c) for c in cache_res]
expr_code = u' + '.join(gen_sig_code(cache_spec))
- code = u'if len(s) == %d:\n return %s\n' % (slen, expr_code)
+ signature_id_tuple = '(%s)' % (
+ ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
+ code = (u'if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
+ u' return %s\n') % (signature_id_tuple, expr_code)
self.to_screen(u'Extracted signature function:\n' + code)
def _parse_sig_js(self, jscode):
return lambda s: initial_function([s])
def _parse_sig_swf(self, file_contents):
- if file_contents[1:3] != b'WS':
- raise ExtractorError(
- u'Not an SWF file; header is %r' % file_contents[:3])
- if file_contents[:1] == b'C':
- content = zlib.decompress(file_contents[8:])
- else:
- raise NotImplementedError(u'Unsupported compression format %r' %
- file_contents[:1])
-
- def extract_tags(content):
- pos = 0
- while pos < len(content):
- header16 = struct.unpack('<H', content[pos:pos+2])[0]
- pos += 2
- tag_code = header16 >> 6
- tag_len = header16 & 0x3f
- if tag_len == 0x3f:
- tag_len = struct.unpack('<I', content[pos:pos+4])[0]
- pos += 4
- assert pos+tag_len <= len(content)
- yield (tag_code, content[pos:pos+tag_len])
- pos += tag_len
-
- code_tag = next(tag
- for tag_code, tag in extract_tags(content)
- if tag_code == 82)
- p = code_tag.index(b'\0', 4) + 1
- code_reader = io.BytesIO(code_tag[p:])
-
- # Parse ABC (AVM2 ByteCode)
- def read_int(reader=None):
- if reader is None:
- reader = code_reader
- res = 0
- shift = 0
- for _ in range(5):
- buf = reader.read(1)
- assert len(buf) == 1
- b = struct.unpack('<B', buf)[0]
- res = res | ((b & 0x7f) << shift)
- if b & 0x80 == 0:
- break
- shift += 7
- return res
-
- def u30(reader=None):
- res = read_int(reader)
- assert res & 0xf0000000 == 0
- return res
- u32 = read_int
-
- def s32(reader=None):
- v = read_int(reader)
- if v & 0x80000000 != 0:
- v = - ((v ^ 0xffffffff) + 1)
- return v
-
- def read_string(reader=None):
- if reader is None:
- reader = code_reader
- slen = u30(reader)
- resb = reader.read(slen)
- assert len(resb) == slen
- return resb.decode('utf-8')
-
- def read_bytes(count, reader=None):
- if reader is None:
- reader = code_reader
- resb = reader.read(count)
- assert len(resb) == count
- return resb
-
- def read_byte(reader=None):
- resb = read_bytes(1, reader=reader)
- res = struct.unpack('<B', resb)[0]
- return res
-
- # minor_version + major_version
- read_bytes(2 + 2)
-
- # Constant pool
- int_count = u30()
- for _c in range(1, int_count):
- s32()
- uint_count = u30()
- for _c in range(1, uint_count):
- u32()
- double_count = u30()
- read_bytes((double_count-1) * 8)
- string_count = u30()
- constant_strings = [u'']
- for _c in range(1, string_count):
- s = read_string()
- constant_strings.append(s)
- namespace_count = u30()
- for _c in range(1, namespace_count):
- read_bytes(1) # kind
- u30() # name
- ns_set_count = u30()
- for _c in range(1, ns_set_count):
- count = u30()
- for _c2 in range(count):
- u30()
- multiname_count = u30()
- MULTINAME_SIZES = {
- 0x07: 2, # QName
- 0x0d: 2, # QNameA
- 0x0f: 1, # RTQName
- 0x10: 1, # RTQNameA
- 0x11: 0, # RTQNameL
- 0x12: 0, # RTQNameLA
- 0x09: 2, # Multiname
- 0x0e: 2, # MultinameA
- 0x1b: 1, # MultinameL
- 0x1c: 1, # MultinameLA
- }
- multinames = [u'']
- for _c in range(1, multiname_count):
- kind = u30()
- assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
- if kind == 0x07:
- u30() # namespace_idx
- name_idx = u30()
- multinames.append(constant_strings[name_idx])
- else:
- multinames.append('[MULTINAME kind: %d]' % kind)
- for _c2 in range(MULTINAME_SIZES[kind]):
- u30()
-
- # Methods
- method_count = u30()
- MethodInfo = collections.namedtuple(
- 'MethodInfo',
- ['NEED_ARGUMENTS', 'NEED_REST'])
- method_infos = []
- for method_id in range(method_count):
- param_count = u30()
- u30() # return type
- for _ in range(param_count):
- u30() # param type
- u30() # name index (always 0 for youtube)
- flags = read_byte()
- if flags & 0x08 != 0:
- # Options present
- option_count = u30()
- for c in range(option_count):
- u30() # val
- read_bytes(1) # kind
- if flags & 0x80 != 0:
- # Param names present
- for _ in range(param_count):
- u30() # param name
- mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
- method_infos.append(mi)
-
- # Metadata
- metadata_count = u30()
- for _c in range(metadata_count):
- u30() # name
- item_count = u30()
- for _c2 in range(item_count):
- u30() # key
- u30() # value
-
- def parse_traits_info():
- trait_name_idx = u30()
- kind_full = read_byte()
- kind = kind_full & 0x0f
- attrs = kind_full >> 4
- methods = {}
- if kind in [0x00, 0x06]: # Slot or Const
- u30() # Slot id
- u30() # type_name_idx
- vindex = u30()
- if vindex != 0:
- read_byte() # vkind
- elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter
- u30() # disp_id
- method_idx = u30()
- methods[multinames[trait_name_idx]] = method_idx
- elif kind == 0x04: # Class
- u30() # slot_id
- u30() # classi
- elif kind == 0x05: # Function
- u30() # slot_id
- function_idx = u30()
- methods[function_idx] = multinames[trait_name_idx]
- else:
- raise ExtractorError(u'Unsupported trait kind %d' % kind)
-
- if attrs & 0x4 != 0: # Metadata present
- metadata_count = u30()
- for _c3 in range(metadata_count):
- u30() # metadata index
-
- return methods
-
- # Classes
+ swfi = SWFInterpreter(file_contents)
TARGET_CLASSNAME = u'SignatureDecipher'
- searched_idx = multinames.index(TARGET_CLASSNAME)
- searched_class_id = None
- class_count = u30()
- for class_id in range(class_count):
- name_idx = u30()
- if name_idx == searched_idx:
- # We found the class we're looking for!
- searched_class_id = class_id
- u30() # super_name idx
- flags = read_byte()
- if flags & 0x08 != 0: # Protected namespace is present
- u30() # protected_ns_idx
- intrf_count = u30()
- for _c2 in range(intrf_count):
- u30()
- u30() # iinit
- trait_count = u30()
- for _c2 in range(trait_count):
- parse_traits_info()
-
- if searched_class_id is None:
- raise ExtractorError(u'Target class %r not found' %
- TARGET_CLASSNAME)
-
- method_names = {}
- method_idxs = {}
- for class_id in range(class_count):
- u30() # cinit
- trait_count = u30()
- for _c2 in range(trait_count):
- trait_methods = parse_traits_info()
- if class_id == searched_class_id:
- method_names.update(trait_methods.items())
- method_idxs.update(dict(
- (idx, name)
- for name, idx in trait_methods.items()))
-
- # Scripts
- script_count = u30()
- for _c in range(script_count):
- u30() # init
- trait_count = u30()
- for _c2 in range(trait_count):
- parse_traits_info()
-
- # Method bodies
- method_body_count = u30()
- Method = collections.namedtuple('Method', ['code', 'local_count'])
- methods = {}
- for _c in range(method_body_count):
- method_idx = u30()
- u30() # max_stack
- local_count = u30()
- u30() # init_scope_depth
- u30() # max_scope_depth
- code_length = u30()
- code = read_bytes(code_length)
- if method_idx in method_idxs:
- m = Method(code, local_count)
- methods[method_idxs[method_idx]] = m
- exception_count = u30()
- for _c2 in range(exception_count):
- u30() # from
- u30() # to
- u30() # target
- u30() # exc_type
- u30() # var_name
- trait_count = u30()
- for _c2 in range(trait_count):
- parse_traits_info()
-
- assert p + code_reader.tell() == len(code_tag)
- assert len(methods) == len(method_idxs)
-
- method_pyfunctions = {}
-
- def extract_function(func_name):
- if func_name in method_pyfunctions:
- return method_pyfunctions[func_name]
- if func_name not in methods:
- raise ExtractorError(u'Cannot find function %r' % func_name)
- m = methods[func_name]
-
- def resfunc(args):
- registers = ['(this)'] + list(args) + [None] * m.local_count
- stack = []
- coder = io.BytesIO(m.code)
- while True:
- opcode = struct.unpack('!B', coder.read(1))[0]
- if opcode == 36: # pushbyte
- v = struct.unpack('!B', coder.read(1))[0]
- stack.append(v)
- elif opcode == 44: # pushstring
- idx = u30(coder)
- stack.append(constant_strings[idx])
- elif opcode == 48: # pushscope
- # We don't implement the scope register, so we'll just
- # ignore the popped value
- stack.pop()
- elif opcode == 70: # callproperty
- index = u30(coder)
- mname = multinames[index]
- arg_count = u30(coder)
- args = list(reversed(
- [stack.pop() for _ in range(arg_count)]))
- obj = stack.pop()
- if mname == u'split':
- assert len(args) == 1
- assert isinstance(args[0], compat_str)
- assert isinstance(obj, compat_str)
- if args[0] == u'':
- res = list(obj)
- else:
- res = obj.split(args[0])
- stack.append(res)
- elif mname == u'slice':
- assert len(args) == 1
- assert isinstance(args[0], int)
- assert isinstance(obj, list)
- res = obj[args[0]:]
- stack.append(res)
- elif mname == u'join':
- assert len(args) == 1
- assert isinstance(args[0], compat_str)
- assert isinstance(obj, list)
- res = args[0].join(obj)
- stack.append(res)
- elif mname in method_pyfunctions:
- stack.append(method_pyfunctions[mname](args))
- else:
- raise NotImplementedError(
- u'Unsupported property %r on %r'
- % (mname, obj))
- elif opcode == 72: # returnvalue
- res = stack.pop()
- return res
- elif opcode == 79: # callpropvoid
- index = u30(coder)
- mname = multinames[index]
- arg_count = u30(coder)
- args = list(reversed(
- [stack.pop() for _ in range(arg_count)]))
- obj = stack.pop()
- if mname == u'reverse':
- assert isinstance(obj, list)
- obj.reverse()
- else:
- raise NotImplementedError(
- u'Unsupported (void) property %r on %r'
- % (mname, obj))
- elif opcode == 93: # findpropstrict
- index = u30(coder)
- mname = multinames[index]
- res = extract_function(mname)
- stack.append(res)
- elif opcode == 97: # setproperty
- index = u30(coder)
- value = stack.pop()
- idx = stack.pop()
- obj = stack.pop()
- assert isinstance(obj, list)
- assert isinstance(idx, int)
- obj[idx] = value
- elif opcode == 98: # getlocal
- index = u30(coder)
- stack.append(registers[index])
- elif opcode == 99: # setlocal
- index = u30(coder)
- value = stack.pop()
- registers[index] = value
- elif opcode == 102: # getproperty
- index = u30(coder)
- pname = multinames[index]
- if pname == u'length':
- obj = stack.pop()
- assert isinstance(obj, list)
- stack.append(len(obj))
- else: # Assume attribute access
- idx = stack.pop()
- assert isinstance(idx, int)
- obj = stack.pop()
- assert isinstance(obj, list)
- stack.append(obj[idx])
- elif opcode == 128: # coerce
- u30(coder)
- elif opcode == 133: # coerce_s
- assert isinstance(stack[-1], (type(None), compat_str))
- elif opcode == 164: # modulo
- value2 = stack.pop()
- value1 = stack.pop()
- res = value1 % value2
- stack.append(res)
- elif opcode == 208: # getlocal_0
- stack.append(registers[0])
- elif opcode == 209: # getlocal_1
- stack.append(registers[1])
- elif opcode == 210: # getlocal_2
- stack.append(registers[2])
- elif opcode == 211: # getlocal_3
- stack.append(registers[3])
- elif opcode == 214: # setlocal_2
- registers[2] = stack.pop()
- elif opcode == 215: # setlocal_3
- registers[3] = stack.pop()
- else:
- raise NotImplementedError(
- u'Unsupported opcode %d' % opcode)
-
- method_pyfunctions[func_name] = resfunc
- return resfunc
-
- initial_function = extract_function(u'decipher')
+ searched_class = swfi.extract_class(TARGET_CLASSNAME)
+ initial_function = swfi.extract_function(searched_class, u'decipher')
return lambda s: initial_function([s])
def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
if player_url.startswith(u'//'):
player_url = u'https:' + player_url
try:
- player_id = (player_url, len(s))
+ player_id = (player_url, self._signature_cache_id(s))
if player_id not in self._player_cache:
func = self._extract_signature_function(
- video_id, player_url, len(s)
+ video_id, player_url, s
)
self._player_cache[player_id] = func
func = self._player_cache[player_id]
if self._downloader.params.get('youtube_print_sig_code'):
- self._print_sig_code(func, len(s))
+ self._print_sig_code(func, s)
return func(s)
except Exception as e:
tb = traceback.format_exc()
raise ExtractorError(
- u'Automatic signature extraction failed: ' + tb, cause=e)
+ u'Signature extraction failed: ' + tb, cause=e)
def _get_available_subtitles(self, video_id, webpage):
try:
sub_lang_list = {}
for l in lang_list:
lang = l[1]
+ if lang in sub_lang_list:
+ continue
params = compat_urllib_parse.urlencode({
'lang': lang,
'v': video_id,
age_gate = True
# We simulate the access to the video from www.youtube.com/v/{video_id}
# this can be viewed without login into Youtube
- data = compat_urllib_parse.urlencode({'video_id': video_id,
- 'el': 'player_embedded',
- 'gl': 'US',
- 'hl': 'en',
- 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
- 'asv': 3,
- 'sts':'1588',
- })
+ data = compat_urllib_parse.urlencode({
+ 'video_id': video_id,
+ 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
+ 'sts': self._search_regex(
+ r'"sts"\s*:\s*(\d+)', video_webpage, 'sts'),
+ })
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
video_info_webpage = self._download_webpage(video_info_url, video_id,
note=False,
upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
upload_date = unified_strdate(upload_date)
- m_cat_container = get_element_by_id("eow-category", video_webpage)
+ m_cat_container = self._search_regex(
+ r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
+ video_webpage, 'categories', fatal=False)
if m_cat_container:
category = self._html_search_regex(
r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
else:
video_description = u''
- def _extract_count(klass):
+ def _extract_count(count_name):
count = self._search_regex(
- r'class="%s">([\d,]+)</span>' % re.escape(klass),
- video_webpage, klass, default=None)
+ r'id="watch-%s"[^>]*>.*?([\d,]+)\s*</span>' % re.escape(count_name),
+ video_webpage, count_name, default=None)
if count is not None:
return int(count.replace(',', ''))
return None
- like_count = _extract_count(u'likes-count')
- dislike_count = _extract_count(u'dislikes-count')
+ like_count = _extract_count(u'like')
+ dislike_count = _extract_count(u'dislike')
# subtitles
video_subtitles = self.extract_subtitles(video_id, video_webpage)
url_map = {}
for url_data_str in encoded_url_map.split(','):
url_data = compat_parse_qs(url_data_str)
- if 'itag' in url_data and 'url' in url_data:
- url = url_data['url'][0]
- if 'sig' in url_data:
- url += '&signature=' + url_data['sig'][0]
- elif 's' in url_data:
- encrypted_sig = url_data['s'][0]
- if self._downloader.params.get('verbose'):
- if age_gate:
- if player_url is None:
- player_version = 'unknown'
- else:
- player_version = self._search_regex(
- r'-(.+)\.swf$', player_url,
- u'flash player', fatal=False)
+ if 'itag' not in url_data or 'url' not in url_data:
+ continue
+ format_id = url_data['itag'][0]
+ url = url_data['url'][0]
+
+ if 'sig' in url_data:
+ url += '&signature=' + url_data['sig'][0]
+ elif 's' in url_data:
+ encrypted_sig = url_data['s'][0]
+
+ if not age_gate:
+ jsplayer_url_json = self._search_regex(
+ r'"assets":.+?"js":\s*("[^"]+")',
+ video_webpage, u'JS player URL')
+ player_url = json.loads(jsplayer_url_json)
+ if player_url is None:
+ player_url_json = self._search_regex(
+ r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
+ video_webpage, u'age gate player URL')
+ player_url = json.loads(player_url_json)
+
+ if self._downloader.params.get('verbose'):
+ if player_url is None:
+ player_version = 'unknown'
+ player_desc = 'unknown'
+ else:
+ if player_url.endswith('swf'):
+ player_version = self._search_regex(
+ r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
+ u'flash player', fatal=False)
player_desc = 'flash player %s' % player_version
else:
player_version = self._search_regex(
- r'html5player-(.+?)\.js', video_webpage,
+ r'html5player-([^/]+?)(?:/html5player)?\.js',
+ player_url,
'html5 player', fatal=False)
player_desc = u'html5 player %s' % player_version
- parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
- self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
- (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
-
- if not age_gate:
- jsplayer_url_json = self._search_regex(
- r'"assets":.+?"js":\s*("[^"]+")',
- video_webpage, u'JS player URL')
- player_url = json.loads(jsplayer_url_json)
-
- signature = self._decrypt_signature(
- encrypted_sig, video_id, player_url, age_gate)
- url += '&signature=' + signature
- if 'ratebypass' not in url:
- url += '&ratebypass=yes'
- url_map[url_data['itag'][0]] = url
+ parts_sizes = self._signature_cache_id(encrypted_sig)
+ self.to_screen(u'{%s} signature length %s, %s' %
+ (format_id, parts_sizes, player_desc))
+
+ signature = self._decrypt_signature(
+ encrypted_sig, video_id, player_url, age_gate)
+ url += '&signature=' + signature
+ if 'ratebypass' not in url:
+ url += '&ratebypass=yes'
+ url_map[format_id] = url
formats = _map_to_format_list(url_map)
elif video_info.get('hlsvp'):
manifest_url = video_info['hlsvp'][0]
self._login()
def _ids_to_results(self, ids):
- return [self.url_result(vid_id, 'Youtube', video_id=vid_id)
- for vid_id in ids]
+ return [
+ self.url_result(vid_id, 'Youtube', video_id=vid_id)
+ for vid_id in ids]
def _extract_mix(self, playlist_id):
# The mixes are generated from a a single video
# the id of the playlist is just 'RD' + video_id
url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
- webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix')
+ webpage = self._download_webpage(
+ url, playlist_id, u'Downloading Youtube mix')
search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
- title_span = (search_title('playlist-title') or
- search_title('title long-title') or search_title('title'))
+ title_span = (
+ search_title('playlist-title') or
+ search_title('title long-title') or
+ search_title('title'))
title = clean_html(title_span)
- video_re = r'''(?x)data-video-username=".*?".*?
- href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re.escape(playlist_id)
- ids = orderedSet(re.findall(video_re, webpage, flags=re.DOTALL))
+ ids = orderedSet(re.findall(
+ r'''(?xs)data-video-username=".*?".*?
+ href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re.escape(playlist_id),
+ webpage))
url_results = self._ids_to_results(ids)
return self.playlist_result(url_results, playlist_id, title)
msg = u'Downloading Youtube mix'
if i > 0:
msg += ', retry #%d' % i
+
webpage = self._download_webpage(url, title, msg)
ids = orderedSet(re.findall(video_re, webpage))
if ids:
paging = mobj.group('paging')
return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
-class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
- IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
- _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
- _FEED_NAME = 'subscriptions'
- _PLAYLIST_TITLE = u'Youtube Subscriptions'
-
class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
_VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
return self.url_result(playlist_id, 'YoutubePlaylist')
+class YoutubeSubscriptionsIE(YoutubePlaylistIE):
+ IE_NAME = u'youtube:subscriptions'
+ IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
+ _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
+
+ def _real_extract(self, url):
+ title = u'Youtube Subscriptions'
+ page = self._download_webpage('https://www.youtube.com/feed/subscriptions', title)
+
+ # The extraction process is the same as for playlists, but the regex
+ # for the video ids doesn't contain an index
+ ids = []
+ more_widget_html = content_html = page
+
+ for page_num in itertools.count(1):
+ matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
+ new_ids = orderedSet(matches)
+ ids.extend(new_ids)
+
+ mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+ if not mobj:
+ break
+
+ more = self._download_json(
+ 'https://youtube.com/%s' % mobj.group('more'), title,
+ 'Downloading page #%s' % page_num,
+ transform_source=uppercase_escape)
+ content_html = more['content_html']
+ more_widget_html = more['load_more_widget_html']
+
+ return {
+ '_type': 'playlist',
+ 'title': title,
+ 'entries': self._ids_to_results(ids),
+ }
+
+
class YoutubeTruncatedURLIE(InfoExtractor):
IE_NAME = 'youtube:truncated_url'
IE_DESC = False # Do not list
from __future__ import unicode_literals
+import json
import re
from .utils import (
assign = lambda v: v
expr = stmt[len('return '):]
else:
- raise ExtractorError(
- 'Cannot determine left side of statement in %r' % stmt)
+ # Try interpreting it as an expression
+ expr = stmt
+ assign = lambda v: v
v = self.interpret_expression(expr, local_vars, allow_recursion)
return assign(v)
if expr.isalpha():
return local_vars[expr]
- m = re.match(r'^(?P<in>[a-z]+)\.(?P<member>.*)$', expr)
+ try:
+ return json.loads(expr)
+ except ValueError:
+ pass
+
+ m = re.match(
+ r'^(?P<var>[a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
+ expr)
if m:
+ variable = m.group('var')
member = m.group('member')
- variable = m.group('in')
+ arg_str = m.group('args')
- if variable not in local_vars:
+ if variable in local_vars:
+ obj = local_vars[variable]
+ else:
if variable not in self._objects:
self._objects[variable] = self.extract_object(variable)
obj = self._objects[variable]
- key, args = member.split('(', 1)
- args = args.strip(')')
- argvals = [int(v) if v.isdigit() else local_vars[v]
- for v in args.split(',')]
- return obj[key](argvals)
-
- val = local_vars[variable]
- if member == 'split("")':
- return list(val)
- if member == 'join("")':
- return ''.join(val)
- if member == 'length':
- return len(val)
- if member == 'reverse()':
- return val[::-1]
- slice_m = re.match(r'slice\((?P<idx>.*)\)', member)
- if slice_m:
- idx = self.interpret_expression(
- slice_m.group('idx'), local_vars, allow_recursion - 1)
- return val[idx:]
+
+ if arg_str is None:
+ # Member access
+ if member == 'length':
+ return len(obj)
+ return obj[member]
+
+ assert expr.endswith(')')
+ # Function call
+ if arg_str == '':
+ argvals = tuple()
+ else:
+ argvals = tuple([
+ self.interpret_expression(v, local_vars, allow_recursion)
+ for v in arg_str.split(',')])
+
+ if member == 'split':
+ assert argvals == ('',)
+ return list(obj)
+ if member == 'join':
+ assert len(argvals) == 1
+ return argvals[0].join(obj)
+ if member == 'reverse':
+ assert len(argvals) == 0
+ obj.reverse()
+ return obj
+ if member == 'slice':
+ assert len(argvals) == 1
+ return obj[argvals[0]:]
+ if member == 'splice':
+ assert isinstance(obj, list)
+ index, howMany = argvals
+ res = []
+ for i in range(index, min(index + howMany, len(obj))):
+ res.append(obj.pop(index))
+ return res
+
+ return obj[member](argvals)
m = re.match(
r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
r'^(?P<func>[a-zA-Z$]+)\((?P<args>[a-z0-9,]+)\)$', expr)
if m:
fname = m.group('func')
+ argvals = tuple([
+ int(v) if v.isdigit() else local_vars[v]
+ for v in m.group('args').split(',')])
if fname not in self._functions:
self._functions[fname] = self.extract_function(fname)
- argvals = [int(v) if v.isdigit() else local_vars[v]
- for v in m.group('args').split(',')]
return self._functions[fname](argvals)
raise ExtractorError('Unsupported JS expression %r' % expr)
obj = {}
obj_m = re.search(
(r'(?:var\s+)?%s\s*=\s*\{' % re.escape(objname)) +
- r'\s*(?P<fields>([a-zA-Z$]+\s*:\s*function\(.*?\)\s*\{.*?\})*)' +
+ r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\})*)' +
r'\}\s*;',
self.code)
fields = obj_m.group('fields')
# Currently, it only supports function definitions
fields_m = re.finditer(
- r'(?P<key>[a-zA-Z$]+)\s*:\s*function'
+ r'(?P<key>[a-zA-Z$0-9]+)\s*:\s*function'
r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
fields)
for f in fields_m:
FFmpegEmbedSubtitlePP,
)
from .xattrpp import XAttrMetadataPP
+from .execafterdownload import ExecAfterDownloadPP
__all__ = [
'AtomicParsleyPP',
'FFmpegExtractAudioPP',
'FFmpegEmbedSubtitlePP',
'XAttrMetadataPP',
+ 'ExecAfterDownloadPP',
]
--- /dev/null
+from __future__ import unicode_literals
+
+import subprocess
+
+from .common import PostProcessor
+from ..utils import (
+ shlex_quote,
+ PostProcessingError,
+)
+
+
+class ExecAfterDownloadPP(PostProcessor):
+ def __init__(self, downloader=None, verboseOutput=None, exec_cmd=None):
+ self.verboseOutput = verboseOutput
+ self.exec_cmd = exec_cmd
+
+ def run(self, information):
+ cmd = self.exec_cmd
+ if not '{}' in cmd:
+ cmd += ' {}'
+
+ cmd = cmd.replace('{}', shlex_quote(information['filepath']))
+
+ self._downloader.to_screen("[exec] Executing command: %s" % cmd)
+ retCode = subprocess.call(cmd, shell=True)
+ if retCode != 0:
+ raise PostProcessingError(
+ 'Command returned error code %d' % retCode)
+
+ return None, information # by default, keep file and do nothing
+
)
-
class FFmpegPostProcessorError(PostProcessingError):
pass
+
class FFmpegPostProcessor(PostProcessor):
- def __init__(self,downloader=None):
+ def __init__(self, downloader=None, deletetempfiles=False):
PostProcessor.__init__(self, downloader)
self._exes = self.detect_executables()
+ self._deletetempfiles = deletetempfiles
@staticmethod
def detect_executables():
stderr = stderr.decode('utf-8', 'replace')
msg = stderr.strip().split('\n')[-1]
raise FFmpegPostProcessorError(msg)
+ if self._deletetempfiles:
+ for ipath in input_paths:
+ os.remove(ipath)
def run_ffmpeg(self, path, out_path, opts):
self.run_ffmpeg_multiple_files([path], out_path, opts)
--- /dev/null
+from __future__ import unicode_literals
+
+import collections
+import io
+import zlib
+
+from .utils import (
+ compat_str,
+ ExtractorError,
+ struct_unpack,
+)
+
+
+def _extract_tags(file_contents):
+ if file_contents[1:3] != b'WS':
+ raise ExtractorError(
+ 'Not an SWF file; header is %r' % file_contents[:3])
+ if file_contents[:1] == b'C':
+ content = zlib.decompress(file_contents[8:])
+ else:
+ raise NotImplementedError(
+ 'Unsupported compression format %r' %
+ file_contents[:1])
+
+ # Determine number of bits in framesize rectangle
+ framesize_nbits = struct_unpack('!B', content[:1])[0] >> 3
+ framesize_len = (5 + 4 * framesize_nbits + 7) // 8
+
+ pos = framesize_len + 2 + 2
+ while pos < len(content):
+ header16 = struct_unpack('<H', content[pos:pos + 2])[0]
+ pos += 2
+ tag_code = header16 >> 6
+ tag_len = header16 & 0x3f
+ if tag_len == 0x3f:
+ tag_len = struct_unpack('<I', content[pos:pos + 4])[0]
+ pos += 4
+ assert pos + tag_len <= len(content), \
+ ('Tag %d ends at %d+%d - that\'s longer than the file (%d)'
+ % (tag_code, pos, tag_len, len(content)))
+ yield (tag_code, content[pos:pos + tag_len])
+ pos += tag_len
+
+
+class _AVMClass_Object(object):
+ def __init__(self, avm_class):
+ self.avm_class = avm_class
+
+ def __repr__(self):
+ return '%s#%x' % (self.avm_class.name, id(self))
+
+
+class _ScopeDict(dict):
+ def __init__(self, avm_class):
+ super(_ScopeDict, self).__init__()
+ self.avm_class = avm_class
+
+ def __repr__(self):
+ return '%s__Scope(%s)' % (
+ self.avm_class.name,
+ super(_ScopeDict, self).__repr__())
+
+
+class _AVMClass(object):
+ def __init__(self, name_idx, name):
+ self.name_idx = name_idx
+ self.name = name
+ self.method_names = {}
+ self.method_idxs = {}
+ self.methods = {}
+ self.method_pyfunctions = {}
+
+ self.variables = _ScopeDict(self)
+
+ def make_object(self):
+ return _AVMClass_Object(self)
+
+ def __repr__(self):
+ return '_AVMClass(%s)' % (self.name)
+
+ def register_methods(self, methods):
+ self.method_names.update(methods.items())
+ self.method_idxs.update(dict(
+ (idx, name)
+ for name, idx in methods.items()))
+
+
+class _Multiname(object):
+ def __init__(self, kind):
+ self.kind = kind
+
+ def __repr__(self):
+ return '[MULTINAME kind: 0x%x]' % self.kind
+
+
+def _read_int(reader):
+ res = 0
+ shift = 0
+ for _ in range(5):
+ buf = reader.read(1)
+ assert len(buf) == 1
+ b = struct_unpack('<B', buf)[0]
+ res = res | ((b & 0x7f) << shift)
+ if b & 0x80 == 0:
+ break
+ shift += 7
+ return res
+
+
+def _u30(reader):
+ res = _read_int(reader)
+ assert res & 0xf0000000 == 0
+ return res
+_u32 = _read_int
+
+
+def _s32(reader):
+ v = _read_int(reader)
+ if v & 0x80000000 != 0:
+ v = - ((v ^ 0xffffffff) + 1)
+ return v
+
+
+def _s24(reader):
+ bs = reader.read(3)
+ assert len(bs) == 3
+ last_byte = b'\xff' if (ord(bs[2:3]) >= 0x80) else b'\x00'
+ return struct_unpack('<i', bs + last_byte)[0]
+
+
+def _read_string(reader):
+ slen = _u30(reader)
+ resb = reader.read(slen)
+ assert len(resb) == slen
+ return resb.decode('utf-8')
+
+
+def _read_bytes(count, reader):
+ assert count >= 0
+ resb = reader.read(count)
+ assert len(resb) == count
+ return resb
+
+
+def _read_byte(reader):
+ resb = _read_bytes(1, reader=reader)
+ res = struct_unpack('<B', resb)[0]
+ return res
+
+
+class SWFInterpreter(object):
+ def __init__(self, file_contents):
+ code_tag = next(tag
+ for tag_code, tag in _extract_tags(file_contents)
+ if tag_code == 82)
+ p = code_tag.index(b'\0', 4) + 1
+ code_reader = io.BytesIO(code_tag[p:])
+
+ # Parse ABC (AVM2 ByteCode)
+
+ # Define a couple convenience methods
+ u30 = lambda *args: _u30(*args, reader=code_reader)
+ s32 = lambda *args: _s32(*args, reader=code_reader)
+ u32 = lambda *args: _u32(*args, reader=code_reader)
+ read_bytes = lambda *args: _read_bytes(*args, reader=code_reader)
+ read_byte = lambda *args: _read_byte(*args, reader=code_reader)
+
+ # minor_version + major_version
+ read_bytes(2 + 2)
+
+ # Constant pool
+ int_count = u30()
+ for _c in range(1, int_count):
+ s32()
+ uint_count = u30()
+ for _c in range(1, uint_count):
+ u32()
+ double_count = u30()
+ read_bytes(max(0, (double_count - 1)) * 8)
+ string_count = u30()
+ self.constant_strings = ['']
+ for _c in range(1, string_count):
+ s = _read_string(code_reader)
+ self.constant_strings.append(s)
+ namespace_count = u30()
+ for _c in range(1, namespace_count):
+ read_bytes(1) # kind
+ u30() # name
+ ns_set_count = u30()
+ for _c in range(1, ns_set_count):
+ count = u30()
+ for _c2 in range(count):
+ u30()
+ multiname_count = u30()
+ MULTINAME_SIZES = {
+ 0x07: 2, # QName
+ 0x0d: 2, # QNameA
+ 0x0f: 1, # RTQName
+ 0x10: 1, # RTQNameA
+ 0x11: 0, # RTQNameL
+ 0x12: 0, # RTQNameLA
+ 0x09: 2, # Multiname
+ 0x0e: 2, # MultinameA
+ 0x1b: 1, # MultinameL
+ 0x1c: 1, # MultinameLA
+ }
+ self.multinames = ['']
+ for _c in range(1, multiname_count):
+ kind = u30()
+ assert kind in MULTINAME_SIZES, 'Invalid multiname kind %r' % kind
+ if kind == 0x07:
+ u30() # namespace_idx
+ name_idx = u30()
+ self.multinames.append(self.constant_strings[name_idx])
+ else:
+ self.multinames.append(_Multiname(kind))
+ for _c2 in range(MULTINAME_SIZES[kind]):
+ u30()
+
+ # Methods
+ method_count = u30()
+ MethodInfo = collections.namedtuple(
+ 'MethodInfo',
+ ['NEED_ARGUMENTS', 'NEED_REST'])
+ method_infos = []
+ for method_id in range(method_count):
+ param_count = u30()
+ u30() # return type
+ for _ in range(param_count):
+ u30() # param type
+ u30() # name index (always 0 for youtube)
+ flags = read_byte()
+ if flags & 0x08 != 0:
+ # Options present
+ option_count = u30()
+ for c in range(option_count):
+ u30() # val
+ read_bytes(1) # kind
+ if flags & 0x80 != 0:
+ # Param names present
+ for _ in range(param_count):
+ u30() # param name
+ mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
+ method_infos.append(mi)
+
+ # Metadata
+ metadata_count = u30()
+ for _c in range(metadata_count):
+ u30() # name
+ item_count = u30()
+ for _c2 in range(item_count):
+ u30() # key
+ u30() # value
+
+ def parse_traits_info():
+ trait_name_idx = u30()
+ kind_full = read_byte()
+ kind = kind_full & 0x0f
+ attrs = kind_full >> 4
+ methods = {}
+ if kind in [0x00, 0x06]: # Slot or Const
+ u30() # Slot id
+ u30() # type_name_idx
+ vindex = u30()
+ if vindex != 0:
+ read_byte() # vkind
+ elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter
+ u30() # disp_id
+ method_idx = u30()
+ methods[self.multinames[trait_name_idx]] = method_idx
+ elif kind == 0x04: # Class
+ u30() # slot_id
+ u30() # classi
+ elif kind == 0x05: # Function
+ u30() # slot_id
+ function_idx = u30()
+ methods[function_idx] = self.multinames[trait_name_idx]
+ else:
+ raise ExtractorError('Unsupported trait kind %d' % kind)
+
+ if attrs & 0x4 != 0: # Metadata present
+ metadata_count = u30()
+ for _c3 in range(metadata_count):
+ u30() # metadata index
+
+ return methods
+
+ # Classes
+ class_count = u30()
+ classes = []
+ for class_id in range(class_count):
+ name_idx = u30()
+
+ cname = self.multinames[name_idx]
+ avm_class = _AVMClass(name_idx, cname)
+ classes.append(avm_class)
+
+ u30() # super_name idx
+ flags = read_byte()
+ if flags & 0x08 != 0: # Protected namespace is present
+ u30() # protected_ns_idx
+ intrf_count = u30()
+ for _c2 in range(intrf_count):
+ u30()
+ u30() # iinit
+ trait_count = u30()
+ for _c2 in range(trait_count):
+ trait_methods = parse_traits_info()
+ avm_class.register_methods(trait_methods)
+
+ assert len(classes) == class_count
+ self._classes_by_name = dict((c.name, c) for c in classes)
+
+ for avm_class in classes:
+ u30() # cinit
+ trait_count = u30()
+ for _c2 in range(trait_count):
+ trait_methods = parse_traits_info()
+ avm_class.register_methods(trait_methods)
+
+ # Scripts
+ script_count = u30()
+ for _c in range(script_count):
+ u30() # init
+ trait_count = u30()
+ for _c2 in range(trait_count):
+ parse_traits_info()
+
+ # Method bodies
+ method_body_count = u30()
+ Method = collections.namedtuple('Method', ['code', 'local_count'])
+ for _c in range(method_body_count):
+ method_idx = u30()
+ u30() # max_stack
+ local_count = u30()
+ u30() # init_scope_depth
+ u30() # max_scope_depth
+ code_length = u30()
+ code = read_bytes(code_length)
+ for avm_class in classes:
+ if method_idx in avm_class.method_idxs:
+ m = Method(code, local_count)
+ avm_class.methods[avm_class.method_idxs[method_idx]] = m
+ exception_count = u30()
+ for _c2 in range(exception_count):
+ u30() # from
+ u30() # to
+ u30() # target
+ u30() # exc_type
+ u30() # var_name
+ trait_count = u30()
+ for _c2 in range(trait_count):
+ parse_traits_info()
+
+ assert p + code_reader.tell() == len(code_tag)
+
+ def extract_class(self, class_name):
+ try:
+ return self._classes_by_name[class_name]
+ except KeyError:
+ raise ExtractorError('Class %r not found' % class_name)
+
+ def extract_function(self, avm_class, func_name):
+ if func_name in avm_class.method_pyfunctions:
+ return avm_class.method_pyfunctions[func_name]
+ if func_name in self._classes_by_name:
+ return self._classes_by_name[func_name].make_object()
+ if func_name not in avm_class.methods:
+ raise ExtractorError('Cannot find function %s.%s' % (
+ avm_class.name, func_name))
+ m = avm_class.methods[func_name]
+
+ def resfunc(args):
+ # Helper functions
+ coder = io.BytesIO(m.code)
+ s24 = lambda: _s24(coder)
+ u30 = lambda: _u30(coder)
+
+ registers = [avm_class.variables] + list(args) + [None] * m.local_count
+ stack = []
+ scopes = collections.deque([
+ self._classes_by_name, avm_class.variables])
+ while True:
+ opcode = _read_byte(coder)
+ if opcode == 17: # iftrue
+ offset = s24()
+ value = stack.pop()
+ if value:
+ coder.seek(coder.tell() + offset)
+ elif opcode == 18: # iffalse
+ offset = s24()
+ value = stack.pop()
+ if not value:
+ coder.seek(coder.tell() + offset)
+ elif opcode == 36: # pushbyte
+ v = _read_byte(coder)
+ stack.append(v)
+ elif opcode == 42: # dup
+ value = stack[-1]
+ stack.append(value)
+ elif opcode == 44: # pushstring
+ idx = u30()
+ stack.append(self.constant_strings[idx])
+ elif opcode == 48: # pushscope
+ new_scope = stack.pop()
+ scopes.append(new_scope)
+ elif opcode == 66: # construct
+ arg_count = u30()
+ args = list(reversed(
+ [stack.pop() for _ in range(arg_count)]))
+ obj = stack.pop()
+ res = obj.avm_class.make_object()
+ stack.append(res)
+ elif opcode == 70: # callproperty
+ index = u30()
+ mname = self.multinames[index]
+ arg_count = u30()
+ args = list(reversed(
+ [stack.pop() for _ in range(arg_count)]))
+ obj = stack.pop()
+
+ if isinstance(obj, _AVMClass_Object):
+ func = self.extract_function(obj.avm_class, mname)
+ res = func(args)
+ stack.append(res)
+ continue
+ elif isinstance(obj, _ScopeDict):
+ if mname in obj.avm_class.method_names:
+ func = self.extract_function(obj.avm_class, mname)
+ res = func(args)
+ else:
+ res = obj[mname]
+ stack.append(res)
+ continue
+ elif isinstance(obj, compat_str):
+ if mname == 'split':
+ assert len(args) == 1
+ assert isinstance(args[0], compat_str)
+ if args[0] == '':
+ res = list(obj)
+ else:
+ res = obj.split(args[0])
+ stack.append(res)
+ continue
+ elif isinstance(obj, list):
+ if mname == 'slice':
+ assert len(args) == 1
+ assert isinstance(args[0], int)
+ res = obj[args[0]:]
+ stack.append(res)
+ continue
+ elif mname == 'join':
+ assert len(args) == 1
+ assert isinstance(args[0], compat_str)
+ res = args[0].join(obj)
+ stack.append(res)
+ continue
+ raise NotImplementedError(
+ 'Unsupported property %r on %r'
+ % (mname, obj))
+ elif opcode == 72: # returnvalue
+ res = stack.pop()
+ return res
+ elif opcode == 74: # constructproperty
+ index = u30()
+ arg_count = u30()
+ args = list(reversed(
+ [stack.pop() for _ in range(arg_count)]))
+ obj = stack.pop()
+
+ mname = self.multinames[index]
+ assert isinstance(obj, _AVMClass)
+
+ # We do not actually call the constructor for now;
+ # we just pretend it does nothing
+ stack.append(obj.make_object())
+ elif opcode == 79: # callpropvoid
+ index = u30()
+ mname = self.multinames[index]
+ arg_count = u30()
+ args = list(reversed(
+ [stack.pop() for _ in range(arg_count)]))
+ obj = stack.pop()
+ if mname == 'reverse':
+ assert isinstance(obj, list)
+ obj.reverse()
+ else:
+ raise NotImplementedError(
+ 'Unsupported (void) property %r on %r'
+ % (mname, obj))
+ elif opcode == 86: # newarray
+ arg_count = u30()
+ arr = []
+ for i in range(arg_count):
+ arr.append(stack.pop())
+ arr = arr[::-1]
+ stack.append(arr)
+ elif opcode == 93: # findpropstrict
+ index = u30()
+ mname = self.multinames[index]
+ for s in reversed(scopes):
+ if mname in s:
+ res = s
+ break
+ else:
+ res = scopes[0]
+ stack.append(res[mname])
+ elif opcode == 94: # findproperty
+ index = u30()
+ mname = self.multinames[index]
+ for s in reversed(scopes):
+ if mname in s:
+ res = s
+ break
+ else:
+ res = avm_class.variables
+ stack.append(res)
+ elif opcode == 96: # getlex
+ index = u30()
+ mname = self.multinames[index]
+ for s in reversed(scopes):
+ if mname in s:
+ scope = s
+ break
+ else:
+ scope = avm_class.variables
+ # I cannot find where static variables are initialized
+ # so let's just return None
+ res = scope.get(mname)
+ stack.append(res)
+ elif opcode == 97: # setproperty
+ index = u30()
+ value = stack.pop()
+ idx = self.multinames[index]
+ if isinstance(idx, _Multiname):
+ idx = stack.pop()
+ obj = stack.pop()
+ obj[idx] = value
+ elif opcode == 98: # getlocal
+ index = u30()
+ stack.append(registers[index])
+ elif opcode == 99: # setlocal
+ index = u30()
+ value = stack.pop()
+ registers[index] = value
+ elif opcode == 102: # getproperty
+ index = u30()
+ pname = self.multinames[index]
+ if pname == 'length':
+ obj = stack.pop()
+ assert isinstance(obj, list)
+ stack.append(len(obj))
+ else: # Assume attribute access
+ idx = stack.pop()
+ assert isinstance(idx, int)
+ obj = stack.pop()
+ assert isinstance(obj, list)
+ stack.append(obj[idx])
+ elif opcode == 115: # convert_
+ value = stack.pop()
+ intvalue = int(value)
+ stack.append(intvalue)
+ elif opcode == 128: # coerce
+ u30()
+ elif opcode == 133: # coerce_s
+ assert isinstance(stack[-1], (type(None), compat_str))
+ elif opcode == 160: # add
+ value2 = stack.pop()
+ value1 = stack.pop()
+ res = value1 + value2
+ stack.append(res)
+ elif opcode == 161: # subtract
+ value2 = stack.pop()
+ value1 = stack.pop()
+ res = value1 - value2
+ stack.append(res)
+ elif opcode == 164: # modulo
+ value2 = stack.pop()
+ value1 = stack.pop()
+ res = value1 % value2
+ stack.append(res)
+ elif opcode == 175: # greaterequals
+ value2 = stack.pop()
+ value1 = stack.pop()
+ result = value1 >= value2
+ stack.append(result)
+ elif opcode == 208: # getlocal_0
+ stack.append(registers[0])
+ elif opcode == 209: # getlocal_1
+ stack.append(registers[1])
+ elif opcode == 210: # getlocal_2
+ stack.append(registers[2])
+ elif opcode == 211: # getlocal_3
+ stack.append(registers[3])
+ elif opcode == 212: # setlocal_0
+ registers[0] = stack.pop()
+ elif opcode == 213: # setlocal_1
+ registers[1] = stack.pop()
+ elif opcode == 214: # setlocal_2
+ registers[2] = stack.pop()
+ elif opcode == 215: # setlocal_3
+ registers[3] = stack.pop()
+ else:
+ raise NotImplementedError(
+ 'Unsupported opcode %d' % opcode)
+
+ avm_class.method_pyfunctions[func_name] = resfunc
+ return resfunc
+
import struct
import subprocess
import sys
+import tempfile
import traceback
import xml.etree.ElementTree
import zlib
compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
try:
- from urllib.parse import parse_qs as compat_parse_qs
-except ImportError: # Python 2
- # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
- # Python 2's version is apparently totally broken
- def _unquote(string, encoding='utf-8', errors='replace'):
+ from urllib.parse import unquote as compat_urllib_parse_unquote
+except ImportError:
+ def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
if string == '':
return string
res = string.split('%')
string += pct_sequence.decode(encoding, errors)
return string
+
+try:
+ from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+ # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+ # Python 2's version is apparently totally broken
+
def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
encoding='utf-8', errors='replace'):
qs, _coerce_result = qs, unicode
continue
if len(nv[1]) or keep_blank_values:
name = nv[0].replace('+', ' ')
- name = _unquote(name, encoding=encoding, errors=errors)
+ name = compat_urllib_parse_unquote(
+ name, encoding=encoding, errors=errors)
name = _coerce_result(name)
value = nv[1].replace('+', ' ')
- value = _unquote(value, encoding=encoding, errors=errors)
+ value = compat_urllib_parse_unquote(
+ value, encoding=encoding, errors=errors)
value = _coerce_result(value)
r.append((name, value))
return r
except ImportError: # Python 2.6
from xml.parsers.expat import ExpatError as compat_xml_parse_error
+try:
+ from shlex import quote as shlex_quote
+except ImportError: # Python < 3.3
+ def shlex_quote(s):
+ return "'" + s.replace("'", "'\"'\"'") + "'"
+
+
def compat_ord(c):
if type(c) is int: return c
else: return ord(c)
assert type(s) == type(u'')
print(s)
-# In Python 2.x, json.dump expects a bytestream.
-# In Python 3.x, it writes to a character stream
-if sys.version_info < (3,0):
- def write_json_file(obj, fn):
- with open(fn, 'wb') as f:
- json.dump(obj, f)
-else:
- def write_json_file(obj, fn):
- with open(fn, 'w', encoding='utf-8') as f:
- json.dump(obj, f)
-if sys.version_info >= (2,7):
+def write_json_file(obj, fn):
+ """ Encode obj as JSON and write it to fn, atomically """
+
+ args = {
+ 'suffix': '.tmp',
+ 'prefix': os.path.basename(fn) + '.',
+ 'dir': os.path.dirname(fn),
+ 'delete': False,
+ }
+
+ # In Python 2.x, json.dump expects a bytestream.
+ # In Python 3.x, it writes to a character stream
+ if sys.version_info < (3, 0):
+ args['mode'] = 'wb'
+ else:
+ args.update({
+ 'mode': 'w',
+ 'encoding': 'utf-8',
+ })
+
+ tf = tempfile.NamedTemporaryFile(**args)
+
+ try:
+ with tf:
+ json.dump(obj, tf)
+ os.rename(tf.name, fn)
+ except:
+ try:
+ os.remove(tf.name)
+ except OSError:
+ pass
+ raise
+
+
+if sys.version_info >= (2, 7):
def find_xpath_attr(node, xpath, key, val):
""" Find the xpath xpath[@key=val] """
- assert re.match(r'^[a-zA-Z]+$', key)
- assert re.match(r'^[a-zA-Z0-9@\s:._]*$', val)
+ assert re.match(r'^[a-zA-Z-]+$', key)
+ assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
expr = xpath + u"[@%s='%s']" % (key, val)
return node.find(expr)
else:
replaced.append('{%s}%s' % (ns_map[ns], tag))
return '/'.join(replaced)
-def htmlentity_transform(matchobj):
- """Transforms an HTML entity to a character.
-
- This function receives a match object and is intended to be used with
- the re.sub() function.
- """
- entity = matchobj.group(1)
-
- # Known non-numeric HTML entity
- if entity in compat_html_entities.name2codepoint:
- return compat_chr(compat_html_entities.name2codepoint[entity])
-
- mobj = re.match(u'(?u)#(x?\\d+)', entity)
- if mobj is not None:
- numstr = mobj.group(1)
- if numstr.startswith(u'x'):
- base = 16
- numstr = u'0%s' % numstr
- else:
- base = 10
- return compat_chr(int(numstr, base))
-
- # Unknown entity in name, return its literal representation
- return (u'&%s;' % entity)
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
class BaseHTMLParser(compat_html_parser.HTMLParser):
return res
+def _htmlentity_transform(entity):
+ """Transforms an HTML entity to a character."""
+ # Known non-numeric HTML entity
+ if entity in compat_html_entities.name2codepoint:
+ return compat_chr(compat_html_entities.name2codepoint[entity])
+
+ mobj = re.match(r'#(x?[0-9]+)', entity)
+ if mobj is not None:
+ numstr = mobj.group(1)
+ if numstr.startswith(u'x'):
+ base = 16
+ numstr = u'0%s' % numstr
+ else:
+ base = 10
+ return compat_chr(int(numstr, base))
+
+ # Unknown entity in name, return its literal representation
+ return (u'&%s;' % entity)
+
+
def unescapeHTML(s):
if s is None:
return None
assert type(s) == compat_str
- result = re.sub(r'(?u)&(.+?);', htmlentity_transform, s)
- return result
+ return re.sub(
+ r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
def encodeFilename(s, for_subprocess=False):
return ret
def http_request(self, req):
- for h,v in std_headers.items():
- if h in req.headers:
- del req.headers[h]
- req.add_header(h, v)
+ for h, v in std_headers.items():
+ if h not in req.headers:
+ req.add_header(h, v)
if 'Youtubedl-no-compression' in req.headers:
if 'Accept-encoding' in req.headers:
del req.headers['Accept-encoding']
'%b %dnd %Y %I:%M%p',
'%b %dth %Y %I:%M%p',
'%Y-%m-%d',
+ '%Y/%m/%d',
'%d.%m.%Y',
'%d/%m/%Y',
+ '%d/%m/%y',
'%Y/%m/%d %H:%M:%S',
'%Y-%m-%d %H:%M:%S',
'%d.%m.%Y %H:%M',
return upload_date
def determine_ext(url, default_ext=u'unknown_video'):
+ if url is None:
+ return default_ext
guess = url.partition(u'?')[0].rpartition(u'.')[2]
if re.match(r'^[A-Za-z0-9]+$', guess):
return guess
return bytes(xs)
-def get_cachedir(params={}):
- cache_root = os.environ.get('XDG_CACHE_HOME',
- os.path.expanduser('~/.cache'))
- return params.get('cachedir', os.path.join(cache_root, 'youtube-dl'))
-
-
# Cross-platform file locking
if sys.platform == 'win32':
import ctypes.wintypes
import fcntl
def _lock_file(f, exclusive):
- fcntl.lockf(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
+ fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
def _unlock_file(f):
- fcntl.lockf(f, fcntl.LOCK_UN)
+ fcntl.flock(f, fcntl.LOCK_UN)
class locked_file(object):
return u'%.2f%s' % (converted, suffix)
-def str_to_int(int_str):
- int_str = re.sub(r'[,\.]', u'', int_str)
- return int(int_str)
-
-
def get_term_width():
columns = os.environ.get('COLUMNS', None)
if columns:
return s
+def remove_end(s, end):
+ if s.endswith(end):
+ return s[:-len(end)]
+ return s
+
+
def url_basename(url):
path = compat_urlparse.urlparse(url).path
return path.strip(u'/').split(u'/')[-1]
return "HEAD"
-def int_or_none(v, scale=1, default=None, get_attr=None):
+def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
if get_attr:
if v is not None:
v = getattr(v, get_attr, None)
- return default if v is None else (int(v) // scale)
+ if v == '':
+ v = None
+ return default if v is None else (int(v) * invscale // scale)
+
+
+def str_or_none(v, default=None):
+ return default if v is None else compat_str(v)
+
+def str_to_int(int_str):
+ """ A more relaxed version of int_or_none """
+ if int_str is None:
+ return None
+ int_str = re.sub(r'[,\.\+]', u'', int_str)
+ return int(int_str)
-def float_or_none(v, scale=1, default=None):
- return default if v is None else (float(v) / scale)
+
+def float_or_none(v, scale=1, invscale=1, default=None):
+ return default if v is None else (float(v) * invscale / scale)
def parse_duration(s):
if s is None:
return None
+ s = s.strip()
+
m = re.match(
- r'(?:(?:(?P<hours>[0-9]+)[:h])?(?P<mins>[0-9]+)[:m])?(?P<secs>[0-9]+)s?(?::[0-9]+)?$', s)
+ r'(?i)(?:(?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?(?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?$', s)
if not m:
return None
res = int(m.group('secs'))
res += int(m.group('mins')) * 60
if m.group('hours'):
res += int(m.group('hours')) * 60 * 60
+ if m.group('ms'):
+ res += float(m.group('ms'))
return res
return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
+try:
+ etree_iter = xml.etree.ElementTree.Element.iter
+except AttributeError: # Python <=2.6
+ etree_iter = lambda n: n.findall('.//*')
+
+
def parse_xml(s):
class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
def doctype(self, name, pubid, system):
parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
- return xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
+ tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
+ # Fix up XML parser in Python 2.x
+ if sys.version_info < (3, 0):
+ for n in etree_iter(tree):
+ if n.text is not None:
+ if not isinstance(n.text, compat_str):
+ n.text = n.text.decode('utf-8')
+ return tree
if sys.version_info < (3, 0) and sys.platform == 'win32':
return re.sub(r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?\s*$', r'\1', code)
+def js_to_json(code):
+ def fix_kv(m):
+ key = m.group(2)
+ if key.startswith("'"):
+ assert key.endswith("'")
+ assert '"' not in key
+ key = '"%s"' % key[1:-1]
+ elif not key.startswith('"'):
+ key = '"%s"' % key
+
+ value = m.group(4)
+ if value.startswith("'"):
+ assert value.endswith("'")
+ assert '"' not in value
+ value = '"%s"' % value[1:-1]
+
+ return m.group(1) + key + m.group(3) + value
+
+ res = re.sub(r'''(?x)
+ ([{,]\s*)
+ ("[^"]*"|\'[^\']*\'|[a-z0-9A-Z]+)
+ (:\s*)
+ ([0-9.]+|true|false|"[^"]*"|\'[^\']*\'|\[|\{)
+ ''', fix_kv, code)
+ res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
+ return res
+
+
def qualities(quality_ids):
""" Get a numeric quality value out of a list of possible values """
def q(qid):
-__version__ = '2014.07.15'
+__version__ = '2014.09.06'
projects / youtube-dl / commitdiff