3 from __future__ import unicode_literals
9 from .common import InfoExtractor
10 from .youtube import YoutubeIE
11 from ..compat import (
12 compat_etree_fromstring,
13 compat_urllib_parse_unquote,
15 compat_xml_parse_error,
33 from .commonprotocols import RtmpIE
34 from .brightcove import (
38 from .nbc import NBCSportsVPlayerIE
39 from .ooyala import OoyalaIE
40 from .rutv import RUTVIE
41 from .tvc import TVCIE
42 from .sportbox import SportBoxEmbedIE
43 from .smotri import SmotriIE
44 from .myvi import MyviIE
45 from .condenast import CondeNastIE
46 from .udn import UDNEmbedIE
47 from .senateisvp import SenateISVPIE
48 from .svt import SVTIE
49 from .pornhub import PornHubIE
50 from .xhamster import XHamsterEmbedIE
51 from .tnaflix import TNAFlixNetworkEmbedIE
52 from .drtuber import DrTuberIE
53 from .redtube import RedTubeIE
54 from .vimeo import VimeoIE
55 from .dailymotion import (
59 from .onionstudios import OnionStudiosIE
60 from .viewlift import ViewLiftEmbedIE
61 from .mtv import MTVServicesEmbeddedIE
62 from .pladform import PladformIE
63 from .videomore import VideomoreIE
64 from .webcaster import WebcasterFeedIE
65 from .googledrive import GoogleDriveIE
66 from .jwplatform import JWPlatformIE
67 from .digiteka import DigitekaIE
68 from .arkena import ArkenaIE
69 from .instagram import InstagramIE
70 from .liveleak import LiveLeakIE
71 from .threeqsdn import ThreeQSDNIE
72 from .theplatform import ThePlatformIE
73 from .vessel import VesselIE
74 from .kaltura import KalturaIE
75 from .eagleplatform import EaglePlatformIE
76 from .facebook import FacebookIE
77 from .soundcloud import SoundcloudIE
78 from .tunein import TuneInBaseIE
79 from .vbox7 import Vbox7IE
80 from .dbtv import DBTVIE
81 from .piksel import PikselIE
82 from .videa import VideaIE
83 from .twentymin import TwentyMinutenIE
84 from .ustream import UstreamIE
85 from .openload import OpenloadIE
86 from .videopress import VideoPressIE
87 from .rutube import RutubeIE
90 class GenericIE(InfoExtractor):
91 IE_DESC = 'Generic downloader that works on some sites'
95 # Direct link to a video
97 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
98 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
103 'upload_date': '20100513',
106 # Direct link to media delivered compressed (until Accept-Encoding is *)
108 'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
109 'md5': '128c42e68b13950268b648275386fc74',
111 'id': 'FictionJunction-Parallel_Hearts',
113 'title': 'FictionJunction-Parallel_Hearts',
114 'upload_date': '20140522',
116 'expected_warnings': [
117 'URL could be a direct video link, returning it as such.'
119 'skip': 'URL invalid',
121 # Direct download with broken HEAD
123 'url': 'http://ai-radio.org:8000/radio.opus',
130 'skip_download': True, # infinite live stream
132 'expected_warnings': [
133 r'501.*Not Implemented',
137 # Direct link with incorrect MIME type
139 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
140 'md5': '4ccbebe5f36706d85221f204d7eb5913',
142 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
143 'id': '5_Lennart_Poettering_-_Systemd',
145 'title': '5_Lennart_Poettering_-_Systemd',
146 'upload_date': '20141120',
148 'expected_warnings': [
149 'URL could be a direct video link, returning it as such.'
154 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
156 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
157 'title': 'Zero Punctuation',
158 'description': 're:.*groundbreaking video review series.*'
160 'playlist_mincount': 11,
162 # RSS feed with enclosure
164 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
166 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
168 'upload_date': '20150228',
169 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
172 # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
174 'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
178 'title': 'Automatics, robotics and biocybernetics',
179 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
180 'upload_date': '20130627',
181 'formats': 'mincount:16',
182 'subtitles': 'mincount:1',
185 'force_generic_extractor': True,
186 'skip_download': True,
189 # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
191 'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
196 'formats': 'mincount:1',
199 'skip_download': True,
202 # SMIL from https://www.restudy.dk/video/play/id/1637
204 'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
208 'title': 'video_1637',
209 'formats': 'mincount:3',
212 'skip_download': True,
215 # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
217 'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
219 'id': 'smil-service',
221 'title': 'smil-service',
222 'formats': 'mincount:1',
225 'skip_download': True,
228 # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
230 'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
234 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
235 'formats': 'mincount:3',
238 'skip_download': True,
241 # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
243 'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
245 'id': 'mZlp2ctYIUEB',
247 'title': 'Tikibad ontruimd wegens brand',
248 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
249 'thumbnail': r're:^https?://.*\.jpg$',
253 'skip_download': True,
256 # MPD from http://dash-mse-test.appspot.com/media.html
258 'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
259 'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
261 'id': 'car-20120827-manifest',
263 'title': 'car-20120827-manifest',
264 'formats': 'mincount:9',
265 'upload_date': '20130904',
268 'format': 'bestvideo',
271 # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
273 'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
278 'formats': 'mincount:8',
282 'skip_download': True,
284 'skip': 'video gone',
286 # m3u8 served with Content-Type: text/plain
288 'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
293 'upload_date': '20140720',
294 'formats': 'mincount:11',
298 'skip_download': True,
300 'skip': 'video gone',
304 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
308 'upload_date': '20130224',
309 'uploader_id': 'TheVerge',
310 'description': r're:^Chris Ziegler takes a look at the\.*',
311 'uploader': 'The Verge',
312 'title': 'First Firefox OS phones side-by-side',
315 'skip_download': False,
319 # redirect in Refresh HTTP header
320 'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
324 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
325 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
326 'upload_date': '20150917',
327 'uploader_id': 'brtvofficial',
328 'uploader': 'Boiler Room',
331 'skip_download': False,
335 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
336 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
338 'id': '13601338388002',
340 'uploader': 'www.hodiho.fr',
341 'title': 'R\u00e9gis plante sa Jeep',
344 # bandcamp page with custom domain
346 'add_ie': ['Bandcamp'],
347 'url': 'http://bronyrock.com/track/the-pony-mash',
351 'title': 'The Pony Mash',
352 'uploader': 'M_Pallante',
354 'skip': 'There is a limit of 200 free downloads / month for the test song',
357 # embedded brightcove video
358 # it also tests brightcove videos that need to set the 'Referer'
359 # in the http requests
360 'add_ie': ['BrightcoveLegacy'],
361 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
363 'id': '2765128793001',
365 'title': 'Le cours de bourse : l’analyse technique',
366 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
367 'uploader': 'BFM BUSINESS',
370 'skip_download': True,
374 # embedded with itemprop embedURL and video id spelled as `idVideo`
375 'add_id': ['BrightcoveLegacy'],
376 'url': 'http://bfmbusiness.bfmtv.com/mediaplayer/chroniques/olivier-delamarche/',
378 'id': '5255628253001',
380 'title': 'md5:37c519b1128915607601e75a87995fc0',
381 'description': 'md5:37f7f888b434bb8f8cc8dbd4f7a4cf26',
382 'uploader': 'BFM BUSINESS',
383 'uploader_id': '876450612001',
384 'timestamp': 1482255315,
385 'upload_date': '20161220',
388 'skip_download': True,
392 # https://github.com/rg3/youtube-dl/issues/2253
393 'url': 'http://bcove.me/i6nfkrc3',
394 'md5': '0ba9446db037002366bab3b3eb30c88c',
396 'id': '3101154703001',
398 'title': 'Still no power',
399 'uploader': 'thestar.com',
400 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
402 'add_ie': ['BrightcoveLegacy'],
403 'skip': 'video gone',
406 'url': 'http://www.championat.com/video/football/v/87/87499.html',
407 'md5': 'fb973ecf6e4a78a67453647444222983',
409 'id': '3414141473001',
411 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
412 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
413 'uploader': 'Championat',
417 # https://github.com/rg3/youtube-dl/issues/3541
418 'add_ie': ['BrightcoveLegacy'],
419 'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
421 'id': '3866516442001',
423 'title': 'Leer mij vrouwen kennen: Aflevering 1',
424 'description': 'Leer mij vrouwen kennen: Aflevering 1',
425 'uploader': 'SBS Broadcasting',
427 'skip': 'Restricted to Netherlands',
429 'skip_download': True, # m3u8 download
433 # Brightcove with alternative playerID key
434 'url': 'http://www.nature.com/nmeth/journal/v9/n7/fig_tab/nmeth.2062_SV1.html',
436 'id': 'nmeth.2062_SV1',
437 'title': 'Simultaneous multiview imaging of the Drosophila syncytial blastoderm : Quantitative high-speed imaging of entire developing embryos with simultaneous multiview light-sheet microscopy : Nature Methods : Nature Research',
441 'id': '2228375078001',
443 'title': 'nmeth.2062-sv1',
444 'description': 'nmeth.2062-sv1',
445 'timestamp': 1363357591,
446 'upload_date': '20130315',
447 'uploader': 'Nature Publishing Group',
448 'uploader_id': '1964492299001',
453 # Brightcove with UUID in videoPlayer
454 'url': 'http://www8.hp.com/cn/zh/home.html',
456 'id': '5255815316001',
458 'title': 'Sprocket Video - China',
459 'description': 'Sprocket Video - China',
460 'uploader': 'HP-Video Gallery',
461 'timestamp': 1482263210,
462 'upload_date': '20161220',
463 'uploader_id': '1107601872001',
466 'skip_download': True, # m3u8 download
471 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
472 'md5': '166dd577b433b4d4ebfee10b0824d8ff',
474 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
476 'title': '2cc213299525360.mov', # that's what we get
479 'add_ie': ['Ooyala'],
482 # ooyala video embedded with http://player.ooyala.com/iframe.js
483 'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
485 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
487 'title': '"Steve Jobs: Man in the Machine" trailer',
488 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
492 'skip_download': True,
494 'skip': 'movie expired',
498 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
502 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
503 'upload_date': '20140225',
504 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
505 'uploader': 'Tested',
506 'uploader_id': 'testedcom',
508 # No need to test YoutubeIE here
510 'skip_download': True,
515 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
519 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
520 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
522 # HEAD requests lead to endless 301, while GET is OK
523 'expected_warnings': ['301'],
527 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
531 'title': 'Охотское море стало целиком российским',
532 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
536 'skip_download': True,
541 'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
545 'title': 'Дошкольное воспитание',
550 'url': 'http://www.vestifinance.ru/articles/25753',
553 'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
558 'title': 'Госзаказ. День 3',
564 'title': 'Госзаказ. День 2',
570 'title': 'Госзаказ. День 1',
576 'skip_download': True,
581 'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
583 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
585 'title': 'Ужастики, русский трейлер (2015)',
586 'thumbnail': r're:^https?://.*\.jpg$',
592 'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
595 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
597 'playlist_mincount': 7,
598 # This forum does not allow <iframe> syntaxes anymore
599 # Now HTML tags are displayed as-is
600 'skip': 'No videos on this page',
604 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
605 'md5': '65fdff94098e4a607385a60c5177c638',
609 'title': 'Hidden miracles of the natural world',
610 'uploader': 'Louie Schwartzberg',
611 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
614 # nowvideo embed hidden behind percent encoding
616 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
617 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
619 'id': '06e53103ca9aa',
621 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
622 'description': 'No description',
627 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
628 'md5': '7653032cbb25bf6c80d80f217055fa43',
630 'id': '048195-004_PLUS7-F',
633 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
634 'upload_date': '20140320',
637 'skip_download': 'Requires rtmpdump'
639 'skip': 'video gone',
643 'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
647 'title': 'Alcaline, le concert avec Calogero',
648 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
649 'upload_date': '20150226',
650 'timestamp': 1424989860,
655 'skip_download': True,
657 'expected_warnings': [
663 'url': 'http://www.wired.com/2014/04/honda-asimo/',
664 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
666 'id': '53501be369702d3275860000',
668 'title': 'Honda’s New Asimo Robot Is More Human Than Ever',
673 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
674 'md5': '441aeeb82eb72c422c7f14ec533999cd',
676 'id': 'k2mm4bCdJ6CQ2i7c8o2',
678 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
679 'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
681 'uploader_id': 'xgditw',
682 'upload_date': '20140425',
683 'timestamp': 1398441542,
685 'add_ie': ['Dailymotion'],
689 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
693 'title': 'The NBL Auction 2014',
694 'uploader': 'BADMINTON England',
695 'uploader_id': 'BADMINTONEvents',
696 'upload_date': '20140603',
697 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
699 'add_ie': ['Youtube'],
701 'skip_download': True,
706 'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
707 'md5': 'ca1aef97695ef2c1d6973256a57e5252',
709 'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
711 'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
712 'description': 'Two valets share their love for movie star Liam Neesons.',
713 'timestamp': 1349922600,
714 'upload_date': '20121011',
717 # YouTube embed via <data-embed-url="">
719 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
723 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
724 'uploader': 'Gameloft',
725 'uploader_id': 'gameloft',
726 'upload_date': '20140828',
727 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
730 'skip_download': True,
735 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
737 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
739 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
740 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
745 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
747 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
748 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
754 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
759 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
760 'md5': '9d65602bf31c6e20014319c7d07fba27',
762 'id': '5123ea6d5e5a7',
765 'uploader': 'www.handjobhub.com',
766 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
769 # Multiple brightcove videos
770 # https://github.com/rg3/youtube-dl/issues/2283
772 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
774 'id': 'always-never',
775 'title': 'Always / Never - The New Yorker',
779 'extract_flat': False,
780 'skip_download': True,
785 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
786 'md5': '96f09a37e44da40dd083e12d9a683327',
790 'title': 'Ump changes call to ball',
791 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
793 'timestamp': 1401537900,
794 'upload_date': '20140531',
795 'thumbnail': r're:^https?://.*\.jpg$',
800 'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
801 'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
805 'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
806 'description': 'a Paywall Videos video from Remilon',
808 'uploader': 'study.com',
809 'timestamp': 1459678540,
810 'upload_date': '20160403',
811 'filesize': 24687186,
815 'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
816 'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
820 'title': 'Conversation about Hexagonal Rails Part 1',
821 'description': 'a Martin Fowler video from ThoughtWorks',
823 'uploader': 'thoughtworks.wistia.com',
824 'timestamp': 1401832161,
825 'upload_date': '20140603',
828 # Wistia standard embed (async)
830 'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
834 'title': 'Drip Brennan Dunn Workshop',
835 'description': 'a JV Webinars video from getdrip-1',
837 'timestamp': 1463607249,
838 'upload_date': '20160518',
841 'skip_download': True,
846 'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
850 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
851 'uploader': 'Sophos Security',
852 'title': 'Chet Chat 171 - Oct 29, 2014',
853 'upload_date': '20141029',
856 # Soundcloud multiple embeds
858 'url': 'http://www.guitarplayer.com/lessons/1014/legato-workout-one-hour-to-more-fluid-performance---tab/52809',
861 'title': 'Guitar Essentials: Legato Workout—One-Hour to Fluid Performance | TAB + AUDIO',
863 'playlist_mincount': 7,
865 # TuneIn station embed
867 'url': 'http://radiocnrv.com/promouvoir-radio-cnrv/',
872 'location': 'Paris, France',
877 'skip_download': True,
882 'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
886 'upload_date': '20141112',
887 'title': 'Rosetta #CometLanding webcast HL 10',
890 # Another Livestream embed, without 'new.' in URL
892 'url': 'https://www.freespeech.org/',
896 'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
900 'skip_download': True,
905 'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
908 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
910 'playlist_mincount': 2,
914 'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
918 'upload_date': '20141126',
919 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
924 'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
926 'id': '730m_DandD_1901_512k',
928 'uploader': 'www.abc.net.au',
929 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
932 # embedded viddler video
934 'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
938 'uploader': 'deadspin',
939 'title': 'WALL-TO-GORTAT',
940 'timestamp': 1422285291,
941 'upload_date': '20150126',
943 'add_ie': ['Viddler'],
947 'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
951 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
952 'description': 'md5:601cb790edd05908957dae8aaa866465',
953 'upload_date': '20150220',
955 'skip': 'All The Daily Show URLs now redirect to http://www.cc.com/shows/',
959 'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
963 'upload_date': '20150212',
964 'uploader': 'The National Archives UK',
965 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
966 'uploader_id': 'NationalArchives08',
967 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
972 'url': 'http://www.suffolk.edu/sjc/',
976 'title': 'Massachusetts Supreme Judicial Court Oral Arguments',
977 'uploader': 'www.suffolk.edu',
980 'skip_download': True,
985 'url': 'http://www.indiedb.com/games/king-machine/videos',
989 'title': 'king machine trailer 1',
990 'thumbnail': r're:^https?://.*\.jpg$',
995 'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
996 'playlist_mincount': 5,
998 'id': 'aanslagen-kopenhagen',
999 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
1004 'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
1008 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
1011 # Kaltura embed (different embed code)
1013 'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
1017 'upload_date': '20150127',
1018 'uploader_id': 'PremierMedia',
1020 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
1023 # Kaltura embed with single quotes
1025 'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
1030 'timestamp': 1355743100,
1031 'upload_date': '20121217',
1032 'uploader_id': 'batchUser',
1034 'add_ie': ['Kaltura'],
1037 # Kaltura embedded via quoted entry_id
1038 'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
1042 'title': '06_matthew_brender_raj_dutt',
1043 'timestamp': 1466638791,
1044 'upload_date': '20160622',
1046 'add_ie': ['Kaltura'],
1047 'expected_warnings': [
1048 'Could not send HEAD request'
1051 'skip_download': True,
1055 # Kaltura embedded, some fileExt broken (#11480)
1056 'url': 'http://www.cornell.edu/video/nima-arkani-hamed-standard-models-of-particle-physics',
1060 'title': 'Our "Standard Models" of particle physics and cosmology',
1061 'description': 'md5:67ea74807b8c4fea92a6f38d6d323861',
1062 'timestamp': 1321158993,
1063 'upload_date': '20111113',
1064 'uploader_id': 'kps1',
1066 'add_ie': ['Kaltura'],
1068 # Eagle.Platform embed (generic URL)
1070 'url': 'http://lenta.ru/news/2015/03/06/navalny/',
1071 # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
1075 'title': 'Навальный вышел на свободу',
1076 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
1077 'thumbnail': r're:^https?://.*\.jpg$',
1083 # ClipYou (Eagle.Platform) embed (custom URL)
1085 'url': 'http://muz-tv.ru/play/7129/',
1086 # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
1090 'title': "'O Sole Mio",
1091 'thumbnail': r're:^https?://.*\.jpg$',
1098 'url': 'http://muz-tv.ru/kinozal/view/7400/',
1102 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
1103 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
1104 'thumbnail': r're:^https?://.*\.jpg$',
1111 'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
1115 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
1116 'thumbnail': r're:^https?://.*\.png$',
1122 'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
1123 'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
1127 'title': 'Facebook Creates "On This Day" | Crunch Report',
1132 'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
1136 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
1141 # Crooks and Liars embed
1143 'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
1147 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
1148 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
1149 'timestamp': 1428207000,
1150 'upload_date': '20150405',
1151 'uploader': 'Heather',
1154 # Crooks and Liars external embed
1156 'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
1158 'id': 'MTE3MjUtMzQ2MzA',
1160 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
1161 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
1162 'timestamp': 1265032391,
1163 'upload_date': '20100201',
1164 'uploader': 'Heather',
1167 # NBC Sports vplayer embed
1169 'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
1171 'id': 'ln7x1qSThw4k',
1173 'title': "PFT Live: New leader in the 'new-look' defense",
1174 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
1175 'uploader': 'NBCU-SPORTS',
1176 'upload_date': '20140107',
1177 'timestamp': 1389118457,
1182 'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
1183 'md5': '1aa589c675898ae6d37a17913cf68d66',
1185 'id': '701714499682',
1187 'title': 'PREVIEW: On Assignment: David Letterman',
1188 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
1193 'url': 'https://video.udn.com/news/300346',
1194 'md5': 'fd2060e988c326991037b9aff9df21a6',
1198 'title': '中一中男師變性 全校師生力挺',
1199 'thumbnail': r're:^https?://.*\.jpg$',
1203 'skip_download': True,
1208 'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
1210 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
1212 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
1213 'title': 'This is what separates the Excel masters from the wannabes',
1214 'duration': 191.933,
1218 'skip_download': True,
1221 # Brightcove URL in single quotes
1223 'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1224 'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1226 'id': '4255764656001',
1228 'title': 'SN Presents: Russell Martin, World Citizen',
1229 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1230 'uploader': 'Rogers Sportsnet',
1231 'uploader_id': '1704050871',
1232 'upload_date': '20150525',
1233 'timestamp': 1432570283,
1236 # Dailymotion Cloud video
1238 'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
1239 'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
1243 'title': 'Sauvons les abeilles ! - Le débat',
1244 'description': 'md5:d9082128b1c5277987825d684939ca26',
1245 'thumbnail': r're:^https?://.*\.jpe?g$',
1246 'timestamp': 1434970506,
1247 'upload_date': '20150622',
1248 'uploader': 'Public Sénat',
1249 'uploader_id': 'xa9gza',
1252 # OnionStudios embed
1254 'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1258 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1259 'thumbnail': r're:^https?://.*\.jpe?g$',
1260 'uploader': 'ClickHole',
1261 'uploader_id': 'clickhole',
1266 'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1268 'id': '74849a00-85a9-11e1-9660-123139220831',
1270 'title': '#whilewewatch',
1273 # AdobeTVVideo embed
1275 'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1276 'md5': '43662b577c018ad707a63766462b1e87',
1280 'title': 'New experience with Acrobat DC',
1281 'description': 'New experience with Acrobat DC',
1282 'duration': 248.667,
1285 # BrightcoveInPageEmbed embed
1287 'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1289 'id': '4238694884001',
1291 'title': 'Tabletop: Dread, Last Thoughts',
1292 'description': 'Tabletop: Dread, Last Thoughts',
1296 # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
1297 # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
1299 'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
1301 'id': '4785848093001',
1303 'title': 'The Cardinal Pell Interview',
1304 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
1305 'uploader': 'GlobeCast Australia - GlobeStream',
1306 'uploader_id': '2733773828001',
1307 'upload_date': '20160304',
1308 'timestamp': 1457083087,
1312 'skip_download': True,
1315 # Another form of arte.tv embed
1317 'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
1318 'md5': '850bfe45417ddf221288c88a0cffe2e2',
1320 'id': '030273-562_PLUS7-F',
1322 'title': 'ARTE Reportage - Nulle part, en France',
1323 'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
1324 'upload_date': '20160409',
1329 'url': 'http://www.wykop.pl/link/3088787/',
1330 'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
1332 'id': '874_1459135191',
1334 'title': 'Man shows poor quality of new apartment building',
1335 'description': 'The wall is like a sand pile.',
1336 'uploader': 'Lake8737',
1339 # Duplicated embedded video URLs
1341 'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
1343 'id': '149298443_480_16c25b74_2',
1345 'title': 'vs. Blue Orange Spring Game',
1346 'uploader': 'www.hudl.com',
1349 # twitter:player:stream embed
1351 'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
1355 'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
1356 'uploader': 'www.rtl.be',
1360 'skip_download': True,
1363 # twitter:player embed
1365 'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
1366 'md5': 'a3e0df96369831de324f0778e126653c',
1368 'id': '4909620399001',
1370 'title': 'What Do Black Holes Sound Like?',
1371 'description': 'what do black holes sound like',
1372 'upload_date': '20160524',
1373 'uploader_id': '29913724001',
1374 'timestamp': 1464107587,
1375 'uploader': 'TheAtlantic',
1377 'add_ie': ['BrightcoveLegacy'],
1379 # Facebook <iframe> embed
1381 'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
1382 'md5': 'fbcde74f534176ecb015849146dd3aee',
1384 'id': '599637780109885',
1386 'title': 'Facebook video #599637780109885',
1389 # Facebook API embed
1391 'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
1392 'md5': 'a47372ee61b39a7b90287094d447d94e',
1394 'id': '10153467542406923',
1396 'title': 'Facebook video #10153467542406923',
1399 # Wordpress "YouTube Video Importer" plugin
1401 'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
1402 'md5': 'd16797741b560b485194eddda8121b48',
1404 'id': 'HNTXWDXV9Is',
1406 'title': 'Blue Devils Drumline Stanford lot 2016',
1407 'upload_date': '20160627',
1408 'uploader_id': 'GENOCIDE8GENERAL10',
1409 'uploader': 'cylus cyrus',
1413 # video stored on custom kaltura server
1414 'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
1415 'md5': '537617d06e64dfed891fa1593c4b30cc',
1419 'title': 'Elecciones británicas: 5 lecciones para Rajoy',
1420 'description': 'md5:435a89d68b9760b92ce67ed227055f16',
1421 'uploader_id': 'videos.expansion@el-mundo.net',
1422 'upload_date': '20150429',
1423 'timestamp': 1430303472,
1425 'add_ie': ['Kaltura'],
1428 # Non-standard Vimeo embed
1429 'url': 'https://openclassrooms.com/courses/understanding-the-web',
1430 'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
1434 'title': 'Understanding the web - Teaser',
1435 'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
1436 'upload_date': '20151214',
1437 'uploader': 'OpenClassrooms',
1438 'uploader_id': 'openclassrooms',
1440 'add_ie': ['Vimeo'],
1443 # generic vimeo embed that requires original URL passed as Referer
1444 'url': 'http://racing4everyone.eu/2016/07/30/formula-1-2016-round12-germany/',
1445 'only_matching': True,
1448 'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
1449 'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
1451 'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
1453 'title': 'Big Buck Bunny',
1454 'description': 'Royalty free test video',
1455 'timestamp': 1432816365,
1456 'upload_date': '20150528',
1460 'skip_download': True,
1462 'add_ie': [ArkenaIE.ie_key()],
1465 'url': 'http://nova.bg/news/view/2016/08/16/156543/%D0%BD%D0%B0-%D0%BA%D0%BE%D1%81%D1%8A%D0%BC-%D0%BE%D1%82-%D0%B2%D0%B7%D1%80%D0%B8%D0%B2-%D0%BE%D1%82%D1%86%D0%B5%D0%BF%D0%B8%D1%85%D0%B0-%D1%86%D1%8F%D0%BB-%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB-%D0%B7%D0%B0%D1%80%D0%B0%D0%B4%D0%B8-%D0%B8%D0%B7%D1%82%D0%B8%D1%87%D0%B0%D0%BD%D0%B5-%D0%BD%D0%B0-%D0%B3%D0%B0%D0%B7-%D0%B2-%D0%BF%D0%BB%D0%BE%D0%B2%D0%B4%D0%B8%D0%B2/',
1469 'title': 'НА КОСЪМ ОТ ВЗРИВ: Изтичане на газ на бензиностанция в Пловдив',
1472 'skip_download': True,
1474 'add_ie': [Vbox7IE.ie_key()],
1478 'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
1481 'title': 'Etter ett års planlegging, klaffet endelig alt: - Jeg måtte ta en liten dans',
1483 'playlist_mincount': 3,
1487 'url': 'http://forum.dvdtalk.com/movie-talk/623756-deleted-magic-star-wars-ot-deleted-alt-scenes-docu-style.html',
1489 'id': '623756-deleted-magic-star-wars-ot-deleted-alt-scenes-docu-style',
1490 'title': 'Deleted Magic - Star Wars: OT Deleted / Alt. Scenes Docu. Style - DVD Talk Forum',
1492 'playlist_mincount': 2,
1496 'url': 'http://www.20min.ch/schweiz/news/story/So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
1500 'title': 'So kommen Sie bei Eis und Schnee sicher an',
1501 'description': 'md5:117c212f64b25e3d95747e5276863f7d',
1504 'skip_download': True,
1506 'add_ie': [TwentyMinutenIE.ie_key()],
1510 'url': 'https://en.support.wordpress.com/videopress/',
1514 'title': 'IMG_5786',
1515 'timestamp': 1435711927,
1516 'upload_date': '20150701',
1519 'skip_download': True,
1521 'add_ie': [VideoPressIE.ie_key()],
1525 'url': 'http://magazzino.friday.ru/videos/vipuski/kazan-2',
1527 'id': '9b3d5bee0a8740bf70dfd29d3ea43541',
1529 'title': 'Магаззино: Казань 2',
1530 'description': 'md5:99bccdfac2269f0e8fdbc4bbc9db184a',
1531 'uploader': 'Магаззино',
1532 'upload_date': '20170228',
1533 'uploader_id': '996642',
1536 'skip_download': True,
1538 'add_ie': [RutubeIE.ie_key()],
1541 # ThePlatform embedded with whitespaces in URLs
1542 'url': 'http://www.golfchannel.com/topics/shows/golftalkcentral.htm',
1543 'only_matching': True,
1546 # # TODO: find another test
1547 # # http://schema.org/VideoObject
1548 # 'url': 'https://flipagram.com/f/nyvTSJMKId',
1549 # 'md5': '888dcf08b7ea671381f00fab74692755',
1551 # 'id': 'nyvTSJMKId',
1553 # 'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
1554 # 'description': '#love for cats.',
1555 # 'timestamp': 1461244995,
1556 # 'upload_date': '20160421',
1559 # 'force_generic_extractor': True,
1564 def report_following_redirect(self, new_url):
1565 """Report information extraction."""
1566 self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
1568 def _extract_rss(self, url, video_id, doc):
1569 playlist_title = doc.find('./channel/title').text
1570 playlist_desc_el = doc.find('./channel/description')
1571 playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
1574 for it in doc.findall('./channel/item'):
1575 next_url = xpath_text(it, 'link', fatal=False)
1577 enclosure_nodes = it.findall('./enclosure')
1578 for e in enclosure_nodes:
1579 next_url = e.attrib.get('url')
1589 'title': it.find('title').text,
1593 '_type': 'playlist',
1595 'title': playlist_title,
1596 'description': playlist_desc,
1600 def _extract_camtasia(self, url, video_id, webpage):
1601 """ Returns None if no camtasia video can be found. """
1603 camtasia_cfg = self._search_regex(
1604 r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
1605 webpage, 'camtasia configuration file', default=None)
1606 if camtasia_cfg is None:
1609 title = self._html_search_meta('DC.title', webpage, fatal=True)
1611 camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
1612 camtasia_cfg = self._download_xml(
1613 camtasia_url, video_id,
1614 note='Downloading camtasia configuration',
1615 errnote='Failed to download camtasia configuration')
1616 fileset_node = camtasia_cfg.find('./playlist/array/fileset')
1619 for n in fileset_node.getchildren():
1620 url_n = n.find('./uri')
1625 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
1626 'title': '%s - %s' % (title, n.tag),
1627 'url': compat_urlparse.urljoin(url, url_n.text),
1628 'duration': float_or_none(n.find('./duration').text),
1632 '_type': 'playlist',
1637 def _real_extract(self, url):
1638 if url.startswith('//'):
1641 'url': self.http_scheme() + url,
1644 parsed_url = compat_urlparse.urlparse(url)
1645 if not parsed_url.scheme:
1646 default_search = self._downloader.params.get('default_search')
1647 if default_search is None:
1648 default_search = 'fixup_error'
1650 if default_search in ('auto', 'auto_warning', 'fixup_error'):
1652 self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
1653 return self.url_result('http://' + url)
1654 elif default_search != 'fixup_error':
1655 if default_search == 'auto_warning':
1656 if re.match(r'^(?:url|URL)$', url):
1657 raise ExtractorError(
1658 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url,
1661 self._downloader.report_warning(
1662 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
1663 return self.url_result('ytsearch:' + url)
1665 if default_search in ('error', 'fixup_error'):
1666 raise ExtractorError(
1667 '%r is not a valid URL. '
1668 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
1669 % (url, url), expected=True)
1671 if ':' not in default_search:
1672 default_search += ':'
1673 return self.url_result(default_search + url)
1675 url, smuggled_data = unsmuggle_url(url)
1676 force_videoid = None
1677 is_intentional = smuggled_data and smuggled_data.get('to_generic')
1678 if smuggled_data and 'force_videoid' in smuggled_data:
1679 force_videoid = smuggled_data['force_videoid']
1680 video_id = force_videoid
1682 video_id = self._generic_id(url)
1684 self.to_screen('%s: Requesting header' % video_id)
1686 head_req = HEADRequest(url)
1687 head_response = self._request_webpage(
1689 note=False, errnote='Could not send HEAD request to %s' % url,
1692 if head_response is not False:
1693 # Check for redirect
1694 new_url = head_response.geturl()
1696 self.report_following_redirect(new_url)
1698 new_url = smuggle_url(
1699 new_url, {'force_videoid': force_videoid})
1700 return self.url_result(new_url)
1702 full_response = None
1703 if head_response is False:
1704 request = sanitized_Request(url)
1705 request.add_header('Accept-Encoding', '*')
1706 full_response = self._request_webpage(request, video_id)
1707 head_response = full_response
1711 'title': self._generic_title(url),
1712 'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
1715 # Check for direct link to a video
1716 content_type = head_response.headers.get('Content-Type', '').lower()
1717 m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
1719 format_id = m.group('format_id')
1720 if format_id.endswith('mpegurl'):
1721 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
1722 elif format_id == 'f4m':
1723 formats = self._extract_f4m_formats(url, video_id)
1726 'format_id': m.group('format_id'),
1728 'vcodec': 'none' if m.group('type') == 'audio' else None
1730 info_dict['direct'] = True
1731 self._sort_formats(formats)
1732 info_dict['formats'] = formats
1735 if not self._downloader.params.get('test', False) and not is_intentional:
1736 force = self._downloader.params.get('force_generic_extractor', False)
1737 self._downloader.report_warning(
1738 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
1740 if not full_response:
1741 request = sanitized_Request(url)
1742 # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
1743 # making it impossible to download only chunk of the file (yet we need only 512kB to
1744 # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
1745 # that will always result in downloading the whole file that is not desirable.
1746 # Therefore for extraction pass we have to override Accept-Encoding to any in order
1747 # to accept raw bytes and being able to download only a chunk.
1748 # It may probably better to solve this by checking Content-Type for application/octet-stream
1749 # after HEAD request finishes, but not sure if we can rely on this.
1750 request.add_header('Accept-Encoding', '*')
1751 full_response = self._request_webpage(request, video_id)
1753 first_bytes = full_response.read(512)
1755 # Is it an M3U playlist?
1756 if first_bytes.startswith(b'#EXTM3U'):
1757 info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
1758 self._sort_formats(info_dict['formats'])
1761 # Maybe it's a direct link to a video?
1762 # Be careful not to download the whole thing!
1763 if not is_html(first_bytes):
1764 self._downloader.report_warning(
1765 'URL could be a direct video link, returning it as such.')
1772 webpage = self._webpage_read_content(
1773 full_response, url, video_id, prefix=first_bytes)
1775 self.report_extraction(video_id)
1777 # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
1779 doc = compat_etree_fromstring(webpage.encode('utf-8'))
1780 if doc.tag == 'rss':
1781 return self._extract_rss(url, video_id, doc)
1782 elif doc.tag == 'SmoothStreamingMedia':
1783 info_dict['formats'] = self._parse_ism_formats(doc, url)
1784 self._sort_formats(info_dict['formats'])
1786 elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
1787 smil = self._parse_smil(doc, url, video_id)
1788 self._sort_formats(smil['formats'])
1790 elif doc.tag == '{http://xspf.org/ns/0/}playlist':
1791 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
1792 elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
1793 info_dict['formats'] = self._parse_mpd_formats(
1795 mpd_base_url=full_response.geturl().rpartition('/')[0],
1797 self._sort_formats(info_dict['formats'])
1799 elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
1800 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
1801 self._sort_formats(info_dict['formats'])
1803 except compat_xml_parse_error:
1806 # Is it a Camtasia project?
1807 camtasia_res = self._extract_camtasia(url, video_id, webpage)
1808 if camtasia_res is not None:
1811 # Sometimes embedded video player is hidden behind percent encoding
1812 # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1813 # Unescaping the whole page allows to handle those cases in a generic way
1814 webpage = compat_urllib_parse_unquote(webpage)
1816 # it's tempting to parse this further, but you would
1817 # have to take into account all the variations like
1818 # Video Title - Site Name
1819 # Site Name | Video Title
1820 # Video Title - Tagline | Site Name
1821 # and so on and so forth; it's just not practical
1822 video_title = self._og_search_title(
1823 webpage, default=None) or self._html_search_regex(
1824 r'(?s)<title>(.*?)</title>', webpage, 'video title',
1827 # Try to detect age limit automatically
1828 age_limit = self._rta_search(webpage)
1829 # And then there are the jokers who advertise that they use RTA,
1830 # but actually don't.
1831 AGE_LIMIT_MARKERS = [
1832 r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1834 if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
1837 # video uploader is domain name
1838 video_uploader = self._search_regex(
1839 r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
1841 video_description = self._og_search_description(webpage, default=None)
1842 video_thumbnail = self._og_search_thumbnail(webpage, default=None)
1844 # Look for Brightcove Legacy Studio embeds
1845 bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
1847 self.to_screen('Brightcove video detected.')
1850 'url': smuggle_url(bc_url, {'Referer': url}),
1851 'ie_key': 'BrightcoveLegacy'
1852 } for bc_url in bc_urls]
1855 '_type': 'playlist',
1856 'title': video_title,
1861 # Look for Brightcove New Studio embeds
1862 bc_urls = BrightcoveNewIE._extract_urls(webpage)
1864 return self.playlist_from_matches(bc_urls, video_id, video_title, ie='BrightcoveNew')
1866 # Look for ThePlatform embeds
1867 tp_urls = ThePlatformIE._extract_urls(webpage)
1869 return self.playlist_from_matches(tp_urls, video_id, video_title, ie='ThePlatform')
1871 # Look for Vessel embeds
1872 vessel_urls = VesselIE._extract_urls(webpage)
1874 return self.playlist_from_matches(vessel_urls, video_id, video_title, ie=VesselIE.ie_key())
1876 # Look for embedded rtl.nl player
1877 matches = re.findall(
1878 r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
1881 return self.playlist_from_matches(matches, video_id, video_title, ie='RtlNl')
1883 vimeo_urls = VimeoIE._extract_urls(url, webpage)
1885 return self.playlist_from_matches(vimeo_urls, video_id, video_title, ie=VimeoIE.ie_key())
1887 vid_me_embed_url = self._search_regex(
1888 r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
1889 webpage, 'vid.me embed', default=None)
1890 if vid_me_embed_url is not None:
1891 return self.url_result(vid_me_embed_url, 'Vidme')
1893 # Look for embedded YouTube player
1894 matches = re.findall(r'''(?x)
1903 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1907 return self.playlist_from_matches(
1908 matches, video_id, video_title, lambda m: unescapeHTML(m[1]))
1910 # Look for lazyYT YouTube embed
1911 matches = re.findall(
1912 r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1914 return self.playlist_from_matches(matches, video_id, video_title, lambda m: unescapeHTML(m))
1916 # Look for Wordpress "YouTube Video Importer" plugin
1917 matches = re.findall(r'''(?x)<div[^>]+
1918 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1919 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1921 return self.playlist_from_matches(matches, video_id, video_title, lambda m: m[-1])
1923 matches = DailymotionIE._extract_urls(webpage)
1925 return self.playlist_from_matches(matches, video_id, video_title)
1927 # Look for embedded Dailymotion playlist player (#3822)
1929 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1931 playlists = re.findall(
1932 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1934 return self.playlist_from_matches(
1935 playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
1937 # Look for embedded Wistia player
1939 r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1941 embed_url = self._proto_relative_url(
1942 unescapeHTML(match.group('url')))
1944 '_type': 'url_transparent',
1947 'uploader': video_uploader,
1950 match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1953 '_type': 'url_transparent',
1954 'url': 'wistia:%s' % match.group('id'),
1956 'uploader': video_uploader,
1961 <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
1962 <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
1965 return self.url_result(self._proto_relative_url(
1966 'wistia:%s' % match.group('id')), 'Wistia')
1968 # Look for SVT player
1969 svt_url = SVTIE._extract_url(webpage)
1971 return self.url_result(svt_url, 'SVT')
1973 # Look for embedded condenast player
1974 matches = re.findall(
1975 r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1979 '_type': 'playlist',
1982 'ie_key': 'CondeNast',
1984 } for ma in matches],
1985 'title': video_title,
1989 # Look for Bandcamp pages with custom domain
1990 mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1991 if mobj is not None:
1992 burl = unescapeHTML(mobj.group(1))
1993 # Don't set the extractor because it can be a track url or an album
1994 return self.url_result(burl)
1996 # Look for embedded Vevo player
1998 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1999 if mobj is not None:
2000 return self.url_result(mobj.group('url'))
2002 # Look for embedded Viddler player
2004 r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
2006 if mobj is not None:
2007 return self.url_result(mobj.group('url'))
2009 # Look for NYTimes player
2011 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
2013 if mobj is not None:
2014 return self.url_result(mobj.group('url'))
2016 # Look for Libsyn player
2018 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
2019 if mobj is not None:
2020 return self.url_result(mobj.group('url'))
2022 # Look for Ooyala videos
2023 mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
2024 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
2025 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
2026 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
2027 if mobj is not None:
2028 embed_token = self._search_regex(
2029 r'embedToken[\'"]?\s*:\s*[\'"]([^\'"]+)',
2030 webpage, 'ooyala embed token', default=None)
2031 return OoyalaIE._build_url_result(smuggle_url(
2034 'embed_token': embed_token,
2037 # Look for multiple Ooyala embeds on SBN network websites
2038 mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
2039 if mobj is not None:
2040 embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
2042 return self.playlist_from_matches(
2043 embeds, video_id, video_title,
2044 getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
2046 # Look for Aparat videos
2047 mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
2048 if mobj is not None:
2049 return self.url_result(mobj.group(1), 'Aparat')
2051 # Look for MPORA videos
2052 mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
2053 if mobj is not None:
2054 return self.url_result(mobj.group(1), 'Mpora')
2056 # Look for embedded NovaMov-based player
2058 r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
2059 (?P<url>http://(?:(?:embed|www)\.)?
2061 nowvideo\.(?:ch|sx|eu|at|ag|co)|
2062 videoweed\.(?:es|com)|
2063 movshare\.(?:net|sx|ag)|
2064 divxstage\.(?:eu|net|ch|co|at|ag))
2065 /embed\.php.+?)\1''', webpage)
2066 if mobj is not None:
2067 return self.url_result(mobj.group('url'))
2069 # Look for embedded Facebook player
2070 facebook_url = FacebookIE._extract_url(webpage)
2071 if facebook_url is not None:
2072 return self.url_result(facebook_url, 'Facebook')
2074 # Look for embedded VK player
2075 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
2076 if mobj is not None:
2077 return self.url_result(mobj.group('url'), 'VK')
2079 # Look for embedded Odnoklassniki player
2080 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
2081 if mobj is not None:
2082 return self.url_result(mobj.group('url'), 'Odnoklassniki')
2084 # Look for embedded ivi player
2085 mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
2086 if mobj is not None:
2087 return self.url_result(mobj.group('url'), 'Ivi')
2089 # Look for embedded Huffington Post player
2091 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
2092 if mobj is not None:
2093 return self.url_result(mobj.group('url'), 'HuffPost')
2096 mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
2097 if mobj is not None:
2098 return self.url_result(mobj.group('url'))
2099 mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
2100 if mobj is not None:
2101 return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
2103 # Look for funnyordie embed
2104 matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
2106 return self.playlist_from_matches(
2107 matches, video_id, video_title, getter=unescapeHTML, ie='FunnyOrDie')
2109 # Look for BBC iPlayer embed
2110 matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
2112 return self.playlist_from_matches(matches, video_id, video_title, ie='BBCCoUk')
2114 # Look for embedded RUTV player
2115 rutv_url = RUTVIE._extract_url(webpage)
2117 return self.url_result(rutv_url, 'RUTV')
2119 # Look for embedded TVC player
2120 tvc_url = TVCIE._extract_url(webpage)
2122 return self.url_result(tvc_url, 'TVC')
2124 # Look for embedded SportBox player
2125 sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
2127 return self.playlist_from_matches(sportbox_urls, video_id, video_title, ie='SportBoxEmbed')
2129 # Look for embedded XHamster player
2130 xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
2132 return self.playlist_from_matches(xhamster_urls, video_id, video_title, ie='XHamsterEmbed')
2134 # Look for embedded TNAFlixNetwork player
2135 tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
2137 return self.playlist_from_matches(tnaflix_urls, video_id, video_title, ie=TNAFlixNetworkEmbedIE.ie_key())
2139 # Look for embedded PornHub player
2140 pornhub_urls = PornHubIE._extract_urls(webpage)
2142 return self.playlist_from_matches(pornhub_urls, video_id, video_title, ie=PornHubIE.ie_key())
2144 # Look for embedded DrTuber player
2145 drtuber_urls = DrTuberIE._extract_urls(webpage)
2147 return self.playlist_from_matches(drtuber_urls, video_id, video_title, ie=DrTuberIE.ie_key())
2149 # Look for embedded RedTube player
2150 redtube_urls = RedTubeIE._extract_urls(webpage)
2152 return self.playlist_from_matches(redtube_urls, video_id, video_title, ie=RedTubeIE.ie_key())
2154 # Look for embedded Tvigle player
2156 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
2157 if mobj is not None:
2158 return self.url_result(mobj.group('url'), 'Tvigle')
2160 # Look for embedded TED player
2162 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
2163 if mobj is not None:
2164 return self.url_result(mobj.group('url'), 'TED')
2166 # Look for embedded Ustream videos
2167 ustream_url = UstreamIE._extract_url(webpage)
2169 return self.url_result(ustream_url, UstreamIE.ie_key())
2171 # Look for embedded arte.tv player
2173 r'<(?:script|iframe) [^>]*?src="(?P<url>http://www\.arte\.tv/(?:playerv2/embed|arte_vp/index)[^"]+)"',
2175 if mobj is not None:
2176 return self.url_result(mobj.group('url'), 'ArteTVEmbed')
2178 # Look for embedded francetv player
2180 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
2182 if mobj is not None:
2183 return self.url_result(mobj.group('url'))
2185 # Look for embedded smotri.com player
2186 smotri_url = SmotriIE._extract_url(webpage)
2188 return self.url_result(smotri_url, 'Smotri')
2190 # Look for embedded Myvi.ru player
2191 myvi_url = MyviIE._extract_url(webpage)
2193 return self.url_result(myvi_url)
2195 # Look for embedded soundcloud player
2196 soundcloud_urls = SoundcloudIE._extract_urls(webpage)
2198 return self.playlist_from_matches(soundcloud_urls, video_id, video_title, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
2200 # Look for tunein player
2201 tunein_urls = TuneInBaseIE._extract_urls(webpage)
2203 return self.playlist_from_matches(tunein_urls, video_id, video_title)
2205 # Look for embedded mtvservices player
2206 mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
2208 return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
2210 # Look for embedded yahoo player
2212 r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
2214 if mobj is not None:
2215 return self.url_result(mobj.group('url'), 'Yahoo')
2217 # Look for embedded sbs.com.au player
2221 <meta\s+property="og:video"\s+content=|
2224 (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
2226 if mobj is not None:
2227 return self.url_result(mobj.group('url'), 'SBS')
2229 # Look for embedded Cinchcast player
2231 r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
2233 if mobj is not None:
2234 return self.url_result(mobj.group('url'), 'Cinchcast')
2237 r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
2241 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
2243 if mobj is not None:
2244 return self.url_result(mobj.group('url'), 'MLB')
2247 r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
2249 if mobj is not None:
2250 return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
2253 r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"',
2255 if mobj is not None:
2256 return self.url_result(mobj.group('url'), 'Livestream')
2258 # Look for Zapiks embed
2260 r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
2261 if mobj is not None:
2262 return self.url_result(mobj.group('url'), 'Zapiks')
2264 # Look for Kaltura embeds
2265 kaltura_url = KalturaIE._extract_url(webpage)
2267 return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
2269 # Look for Eagle.Platform embeds
2270 eagleplatform_url = EaglePlatformIE._extract_url(webpage)
2271 if eagleplatform_url:
2272 return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
2274 # Look for ClipYou (uses Eagle.Platform) embeds
2276 r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
2277 if mobj is not None:
2278 return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
2280 # Look for Pladform embeds
2281 pladform_url = PladformIE._extract_url(webpage)
2283 return self.url_result(pladform_url)
2285 # Look for Videomore embeds
2286 videomore_url = VideomoreIE._extract_url(webpage)
2288 return self.url_result(videomore_url)
2290 # Look for Webcaster embeds
2291 webcaster_url = WebcasterFeedIE._extract_url(self, webpage)
2293 return self.url_result(webcaster_url, ie=WebcasterFeedIE.ie_key())
2295 # Look for Playwire embeds
2297 r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
2298 if mobj is not None:
2299 return self.url_result(mobj.group('url'))
2301 # Look for 5min embeds
2303 r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
2304 if mobj is not None:
2305 return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
2307 # Look for Crooks and Liars embeds
2309 r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
2310 if mobj is not None:
2311 return self.url_result(mobj.group('url'))
2313 # Look for NBC Sports VPlayer embeds
2314 nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
2316 return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
2318 # Look for NBC News embeds
2319 nbc_news_embed_url = re.search(
2320 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1', webpage)
2321 if nbc_news_embed_url:
2322 return self.url_result(nbc_news_embed_url.group('url'), 'NBCNews')
2324 # Look for Google Drive embeds
2325 google_drive_url = GoogleDriveIE._extract_url(webpage)
2326 if google_drive_url:
2327 return self.url_result(google_drive_url, 'GoogleDrive')
2329 # Look for UDN embeds
2331 r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
2332 if mobj is not None:
2333 return self.url_result(
2334 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
2336 # Look for Senate ISVP iframe
2337 senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
2339 return self.url_result(senate_isvp_url, 'SenateISVP')
2341 # Look for Dailymotion Cloud videos
2342 dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
2344 return self.url_result(dmcloud_url, 'DailymotionCloud')
2346 # Look for OnionStudios embeds
2347 onionstudios_url = OnionStudiosIE._extract_url(webpage)
2348 if onionstudios_url:
2349 return self.url_result(onionstudios_url)
2351 # Look for ViewLift embeds
2352 viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
2354 return self.url_result(viewlift_url)
2356 # Look for JWPlatform embeds
2357 jwplatform_url = JWPlatformIE._extract_url(webpage)
2359 return self.url_result(jwplatform_url, 'JWPlatform')
2361 # Look for Digiteka embeds
2362 digiteka_url = DigitekaIE._extract_url(webpage)
2364 return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
2366 # Look for Arkena embeds
2367 arkena_url = ArkenaIE._extract_url(webpage)
2369 return self.url_result(arkena_url, ArkenaIE.ie_key())
2371 # Look for Piksel embeds
2372 piksel_url = PikselIE._extract_url(webpage)
2374 return self.url_result(piksel_url, PikselIE.ie_key())
2376 # Look for Limelight embeds
2377 mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
2381 'Channel': 'channel',
2382 'ChannelList': 'channel_list',
2384 return self.url_result(smuggle_url('limelight:%s:%s' % (
2385 lm[mobj.group(1)], mobj.group(2)), {'source_url': url}),
2386 'Limelight%s' % mobj.group(1), mobj.group(2))
2390 <object[^>]+class=(["\'])LimelightEmbeddedPlayerFlash\1[^>]*>.*?
2392 name=(["\'])flashVars\2[^>]+
2393 value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
2396 return self.url_result(smuggle_url(
2397 'limelight:media:%s' % mobj.group('id'),
2398 {'source_url': url}), 'LimelightMedia', mobj.group('id'))
2400 # Look for AdobeTVVideo embeds
2402 r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
2404 if mobj is not None:
2405 return self.url_result(
2406 self._proto_relative_url(unescapeHTML(mobj.group(1))),
2409 # Look for Vine embeds
2411 r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
2413 if mobj is not None:
2414 return self.url_result(
2415 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
2417 # Look for VODPlatform embeds
2419 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vod-platform\.net/[eE]mbed/.+?)\1',
2421 if mobj is not None:
2422 return self.url_result(
2423 self._proto_relative_url(unescapeHTML(mobj.group('url'))), 'VODPlatform')
2425 # Look for Mangomolo embeds
2427 r'''(?x)<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?admin\.mangomolo\.com/analytics/index\.php/customers/embed/
2429 video\?.*?\bid=(?P<video_id>\d+)|
2430 index\?.*?\bchannelid=(?P<channel_id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)
2431 ).+?)\1''', webpage)
2432 if mobj is not None:
2434 '_type': 'url_transparent',
2435 'url': self._proto_relative_url(unescapeHTML(mobj.group('url'))),
2436 'title': video_title,
2437 'description': video_description,
2438 'thumbnail': video_thumbnail,
2439 'uploader': video_uploader,
2441 video_id = mobj.group('video_id')
2444 'ie_key': 'MangomoloVideo',
2449 'ie_key': 'MangomoloLive',
2450 'id': mobj.group('channel_id'),
2454 # Look for Instagram embeds
2455 instagram_embed_url = InstagramIE._extract_embed_url(webpage)
2456 if instagram_embed_url is not None:
2457 return self.url_result(
2458 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
2460 # Look for LiveLeak embeds
2461 liveleak_url = LiveLeakIE._extract_url(webpage)
2463 return self.url_result(liveleak_url, 'LiveLeak')
2465 # Look for 3Q SDN embeds
2466 threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
2469 '_type': 'url_transparent',
2470 'ie_key': ThreeQSDNIE.ie_key(),
2471 'url': self._proto_relative_url(threeqsdn_url),
2472 'title': video_title,
2473 'description': video_description,
2474 'thumbnail': video_thumbnail,
2475 'uploader': video_uploader,
2478 # Look for VBOX7 embeds
2479 vbox7_url = Vbox7IE._extract_url(webpage)
2481 return self.url_result(vbox7_url, Vbox7IE.ie_key())
2483 # Look for DBTV embeds
2484 dbtv_urls = DBTVIE._extract_urls(webpage)
2486 return self.playlist_from_matches(dbtv_urls, video_id, video_title, ie=DBTVIE.ie_key())
2488 # Look for Videa embeds
2489 videa_urls = VideaIE._extract_urls(webpage)
2491 return self.playlist_from_matches(videa_urls, video_id, video_title, ie=VideaIE.ie_key())
2493 # Look for 20 minuten embeds
2494 twentymin_urls = TwentyMinutenIE._extract_urls(webpage)
2496 return self.playlist_from_matches(
2497 twentymin_urls, video_id, video_title, ie=TwentyMinutenIE.ie_key())
2499 # Look for Openload embeds
2500 openload_urls = OpenloadIE._extract_urls(webpage)
2502 return self.playlist_from_matches(
2503 openload_urls, video_id, video_title, ie=OpenloadIE.ie_key())
2505 # Look for VideoPress embeds
2506 videopress_urls = VideoPressIE._extract_urls(webpage)
2508 return self.playlist_from_matches(
2509 videopress_urls, video_id, video_title, ie=VideoPressIE.ie_key())
2511 # Look for Rutube embeds
2512 rutube_urls = RutubeIE._extract_urls(webpage)
2514 return self.playlist_from_matches(
2515 rutube_urls, ie=RutubeIE.ie_key())
2517 # Looking for http://schema.org/VideoObject
2518 json_ld = self._search_json_ld(
2519 webpage, video_id, default={}, expected_type='VideoObject')
2520 if json_ld.get('url'):
2522 'title': video_title or info_dict['title'],
2523 'description': video_description,
2524 'thumbnail': video_thumbnail,
2525 'age_limit': age_limit
2527 info_dict.update(json_ld)
2530 # Look for HTML5 media
2531 entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
2533 for entry in entries:
2536 'title': video_title,
2538 self._sort_formats(entry['formats'])
2539 return self.playlist_result(entries)
2541 jwplayer_data_str = self._find_jwplayer_data(webpage)
2542 if jwplayer_data_str:
2544 jwplayer_data = self._parse_json(
2545 jwplayer_data_str, video_id, transform_source=js_to_json)
2546 info = self._parse_jwplayer_data(
2547 jwplayer_data, video_id, require_title=False)
2548 if not info.get('title'):
2549 info['title'] = video_title
2551 except ExtractorError:
2554 def check_video(vurl):
2555 if YoutubeIE.suitable(vurl):
2557 if RtmpIE.suitable(vurl):
2559 vpath = compat_urlparse.urlparse(vurl).path
2560 vext = determine_ext(vpath)
2561 return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js')
2563 def filter_video(urls):
2564 return list(filter(check_video, urls))
2566 # Start with something easy: JW Player in SWFObject
2567 found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
2569 # Look for gorilla-vid style embedding
2570 found = filter_video(re.findall(r'''(?sx)
2574 jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
2577 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
2579 # Broaden the search a little bit
2580 found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
2582 # Broaden the findall a little bit: JWPlayer JS loader
2583 found = filter_video(re.findall(
2584 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
2587 found = filter_video(re.findall(r'''(?xs)
2588 flowplayer\("[^"]+",\s*
2590 \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
2591 ["']?url["']?\s*:\s*["']([^"']+)["']
2596 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
2598 # Try to find twitter cards info
2599 # twitter:player:stream should be checked before twitter:player since
2600 # it is expected to contain a raw stream (see
2601 # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2602 found = filter_video(re.findall(
2603 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
2605 # We look for Open Graph info:
2606 # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
2607 m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
2608 # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
2609 if m_video_type is not None:
2610 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
2612 REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
2614 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
2615 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
2618 # Look also in Refresh HTTP header
2619 refresh_header = head_response.headers.get('Refresh')
2621 # In python 2 response HTTP headers are bytestrings
2622 if sys.version_info < (3, 0) and isinstance(refresh_header, str):
2623 refresh_header = refresh_header.decode('iso-8859-1')
2624 found = re.search(REDIRECT_REGEX, refresh_header)
2626 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
2627 self.report_following_redirect(new_url)
2634 # twitter:player is a https URL to iframe player that may or may not
2635 # be supported by youtube-dl thus this is checked the very last (see
2636 # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
2637 embed_url = self._html_search_meta('twitter:player', webpage, default=None)
2639 return self.url_result(embed_url)
2642 raise UnsupportedError(url)
2645 for video_url in orderedSet(found):
2646 video_url = unescapeHTML(video_url)
2647 video_url = video_url.replace('\\/', '/')
2648 video_url = compat_urlparse.urljoin(url, video_url)
2649 video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
2651 # Sometimes, jwplayer extraction will result in a YouTube URL
2652 if YoutubeIE.suitable(video_url):
2653 entries.append(self.url_result(video_url, 'Youtube'))
2656 # here's a fun little line of code for you:
2657 video_id = os.path.splitext(video_id)[0]
2661 'uploader': video_uploader,
2662 'title': video_title,
2663 'age_limit': age_limit,
2666 if RtmpIE.suitable(video_url):
2667 entry_info_dict.update({
2668 '_type': 'url_transparent',
2669 'ie_key': RtmpIE.ie_key(),
2672 entries.append(entry_info_dict)
2675 ext = determine_ext(video_url)
2677 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
2679 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
2681 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
2683 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
2685 entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
2686 elif re.search(r'(?i)\.(?:ism|smil)/manifest', video_url) and video_url != url:
2687 # Just matching .ism/manifest is not enough to be reliably sure
2688 # whether it's actually an ISM manifest or some other streaming
2689 # manifest since there are various streaming URL formats
2690 # possible (see [1]) as well as some other shenanigans like
2691 # .smil/manifest URLs that actually serve an ISM (see [2]) and
2693 # Thus the most reasonable way to solve this is to delegate
2694 # to generic extractor in order to look into the contents of
2695 # the manifest itself.
2696 # 1. https://azure.microsoft.com/en-us/documentation/articles/media-services-deliver-content-overview/#streaming-url-formats
2697 # 2. https://svs.itworkscdn.net/lbcivod/smil:itwfcdn/lbci/170976.smil/Manifest
2698 entry_info_dict = self.url_result(
2699 smuggle_url(video_url, {'to_generic': True}),
2702 entry_info_dict['url'] = video_url
2704 if entry_info_dict.get('formats'):
2705 self._sort_formats(entry_info_dict['formats'])
2707 entries.append(entry_info_dict)
2709 if len(entries) == 1:
2712 for num, e in enumerate(entries, start=1):
2713 # 'url' results don't have a title
2714 if e.get('title') is not None:
2715 e['title'] = '%s (%d)' % (e['title'], num)
2717 '_type': 'playlist',