_ Git - youtube-dl/blob - youtube_dl/extractor/generic.py

   1 # encoding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import os
   6 import re
   7 import sys
   8
   9 from .common import InfoExtractor
  10 from .youtube import YoutubeIE
  11 from ..compat import (
  12     compat_etree_fromstring,
  13     compat_urllib_parse_unquote,
  14     compat_urlparse,
  15     compat_xml_parse_error,
  16 )
  17 from ..utils import (
  18     determine_ext,
  19     ExtractorError,
  20     float_or_none,
  21     HEADRequest,
  22     is_html,
  23     orderedSet,
  24     sanitized_Request,
  25     smuggle_url,
  26     unescapeHTML,
  27     unified_strdate,
  28     unsmuggle_url,
  29     UnsupportedError,
  30     url_basename,
  31     xpath_text,
  32 )
  33 from .brightcove import (
  34     BrightcoveLegacyIE,
  35     BrightcoveNewIE,
  36 )
  37 from .nbc import NBCSportsVPlayerIE
  38 from .ooyala import OoyalaIE
  39 from .rutv import RUTVIE
  40 from .tvc import TVCIE
  41 from .sportbox import SportBoxEmbedIE
  42 from .smotri import SmotriIE
  43 from .myvi import MyviIE
  44 from .condenast import CondeNastIE
  45 from .udn import UDNEmbedIE
  46 from .senateisvp import SenateISVPIE
  47 from .bliptv import BlipTVIE
  48 from .svt import SVTIE
  49 from .pornhub import PornHubIE
  50 from .xhamster import XHamsterEmbedIE
  51 from .vimeo import VimeoIE
  52 from .dailymotion import DailymotionCloudIE
  53 from .onionstudios import OnionStudiosIE
  54 from .snagfilms import SnagFilmsEmbedIE
  55 from .screenwavemedia import ScreenwaveMediaIE
  56 from .mtv import MTVServicesEmbeddedIE
  57 from .pladform import PladformIE
  58
  59
  60 class GenericIE(InfoExtractor):
  61     IE_DESC = 'Generic downloader that works on some sites'
  62     _VALID_URL = r'.*'
  63     IE_NAME = 'generic'
  64     _TESTS = [
  65         # Direct link to a video
  66         {
  67             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
  68             'md5': '67d406c2bcb6af27fa886f31aa934bbe',
  69             'info_dict': {
  70                 'id': 'trailer',
  71                 'ext': 'mp4',
  72                 'title': 'trailer',
  73                 'upload_date': '20100513',
  74             }
  75         },
  76         # Direct link to media delivered compressed (until Accept-Encoding is *)
  77         {
  78             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
  79             'md5': '128c42e68b13950268b648275386fc74',
  80             'info_dict': {
  81                 'id': 'FictionJunction-Parallel_Hearts',
  82                 'ext': 'flac',
  83                 'title': 'FictionJunction-Parallel_Hearts',
  84                 'upload_date': '20140522',
  85             },
  86             'expected_warnings': [
  87                 'URL could be a direct video link, returning it as such.'
  88             ]
  89         },
  90         # Direct download with broken HEAD
  91         {
  92             'url': 'http://ai-radio.org:8000/radio.opus',
  93             'info_dict': {
  94                 'id': 'radio',
  95                 'ext': 'opus',
  96                 'title': 'radio',
  97             },
  98             'params': {
  99                 'skip_download': True,  # infinite live stream
 100             },
 101             'expected_warnings': [
 102                 r'501.*Not Implemented'
 103             ],
 104         },
 105         # Direct link with incorrect MIME type
 106         {
 107             'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 108             'md5': '4ccbebe5f36706d85221f204d7eb5913',
 109             'info_dict': {
 110                 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 111                 'id': '5_Lennart_Poettering_-_Systemd',
 112                 'ext': 'webm',
 113                 'title': '5_Lennart_Poettering_-_Systemd',
 114                 'upload_date': '20141120',
 115             },
 116             'expected_warnings': [
 117                 'URL could be a direct video link, returning it as such.'
 118             ]
 119         },
 120         # RSS feed
 121         {
 122             'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 123             'info_dict': {
 124                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 125                 'title': 'Zero Punctuation',
 126                 'description': 're:.*groundbreaking video review series.*'
 127             },
 128             'playlist_mincount': 11,
 129         },
 130         # RSS feed with enclosure
 131         {
 132             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
 133             'info_dict': {
 134                 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 135                 'ext': 'm4v',
 136                 'upload_date': '20150228',
 137                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 138             }
 139         },
 140         # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
 141         {
 142             'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
 143             'info_dict': {
 144                 'id': 'smil',
 145                 'ext': 'mp4',
 146                 'title': 'Automatics, robotics and biocybernetics',
 147                 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
 148                 'upload_date': '20130627',
 149                 'formats': 'mincount:16',
 150                 'subtitles': 'mincount:1',
 151             },
 152             'params': {
 153                 'force_generic_extractor': True,
 154                 'skip_download': True,
 155             },
 156         },
 157         # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
 158         {
 159             'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
 160             'info_dict': {
 161                 'id': 'hds',
 162                 'ext': 'flv',
 163                 'title': 'hds',
 164                 'formats': 'mincount:1',
 165             },
 166             'params': {
 167                 'skip_download': True,
 168             },
 169         },
 170         # SMIL from https://www.restudy.dk/video/play/id/1637
 171         {
 172             'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
 173             'info_dict': {
 174                 'id': 'video_1637',
 175                 'ext': 'flv',
 176                 'title': 'video_1637',
 177                 'formats': 'mincount:3',
 178             },
 179             'params': {
 180                 'skip_download': True,
 181             },
 182         },
 183         # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
 184         {
 185             'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
 186             'info_dict': {
 187                 'id': 'smil-service',
 188                 'ext': 'flv',
 189                 'title': 'smil-service',
 190                 'formats': 'mincount:1',
 191             },
 192             'params': {
 193                 'skip_download': True,
 194             },
 195         },
 196         # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
 197         {
 198             'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
 199             'info_dict': {
 200                 'id': '4719370',
 201                 'ext': 'mp4',
 202                 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
 203                 'formats': 'mincount:3',
 204             },
 205             'params': {
 206                 'skip_download': True,
 207             },
 208         },
 209         # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
 210         {
 211             'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
 212             'info_dict': {
 213                 'id': 'mZlp2ctYIUEB',
 214                 'ext': 'mp4',
 215                 'title': 'Tikibad ontruimd wegens brand',
 216                 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
 217                 'thumbnail': 're:^https?://.*\.jpg$',
 218                 'duration': 33,
 219             },
 220             'params': {
 221                 'skip_download': True,
 222             },
 223         },
 224         # google redirect
 225         {
 226             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
 227             'info_dict': {
 228                 'id': 'cmQHVoWB5FY',
 229                 'ext': 'mp4',
 230                 'upload_date': '20130224',
 231                 'uploader_id': 'TheVerge',
 232                 'description': 're:^Chris Ziegler takes a look at the\.*',
 233                 'uploader': 'The Verge',
 234                 'title': 'First Firefox OS phones side-by-side',
 235             },
 236             'params': {
 237                 'skip_download': False,
 238             }
 239         },
 240         {
 241             # redirect in Refresh HTTP header
 242             'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
 243             'info_dict': {
 244                 'id': 'pO8h3EaFRdo',
 245                 'ext': 'mp4',
 246                 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
 247                 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
 248                 'upload_date': '20150917',
 249                 'uploader_id': 'brtvofficial',
 250                 'uploader': 'Boiler Room',
 251             },
 252             'params': {
 253                 'skip_download': False,
 254             },
 255         },
 256         {
 257             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
 258             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
 259             'info_dict': {
 260                 'id': '13601338388002',
 261                 'ext': 'mp4',
 262                 'uploader': 'www.hodiho.fr',
 263                 'title': 'R\u00e9gis plante sa Jeep',
 264             }
 265         },
 266         # bandcamp page with custom domain
 267         {
 268             'add_ie': ['Bandcamp'],
 269             'url': 'http://bronyrock.com/track/the-pony-mash',
 270             'info_dict': {
 271                 'id': '3235767654',
 272                 'ext': 'mp3',
 273                 'title': 'The Pony Mash',
 274                 'uploader': 'M_Pallante',
 275             },
 276             'skip': 'There is a limit of 200 free downloads / month for the test song',
 277         },
 278         # embedded brightcove video
 279         # it also tests brightcove videos that need to set the 'Referer' in the
 280         # http requests
 281         {
 282             'add_ie': ['BrightcoveLegacy'],
 283             'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
 284             'info_dict': {
 285                 'id': '2765128793001',
 286                 'ext': 'mp4',
 287                 'title': 'Le cours de bourse : l’analyse technique',
 288                 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
 289                 'uploader': 'BFM BUSINESS',
 290             },
 291             'params': {
 292                 'skip_download': True,
 293             },
 294         },
 295         {
 296             # https://github.com/rg3/youtube-dl/issues/2253
 297             'url': 'http://bcove.me/i6nfkrc3',
 298             'md5': '0ba9446db037002366bab3b3eb30c88c',
 299             'info_dict': {
 300                 'id': '3101154703001',
 301                 'ext': 'mp4',
 302                 'title': 'Still no power',
 303                 'uploader': 'thestar.com',
 304                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
 305             },
 306             'add_ie': ['BrightcoveLegacy'],
 307         },
 308         {
 309             'url': 'http://www.championat.com/video/football/v/87/87499.html',
 310             'md5': 'fb973ecf6e4a78a67453647444222983',
 311             'info_dict': {
 312                 'id': '3414141473001',
 313                 'ext': 'mp4',
 314                 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
 315                 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
 316                 'uploader': 'Championat',
 317             },
 318         },
 319         {
 320             # https://github.com/rg3/youtube-dl/issues/3541
 321             'add_ie': ['BrightcoveLegacy'],
 322             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
 323             'info_dict': {
 324                 'id': '3866516442001',
 325                 'ext': 'mp4',
 326                 'title': 'Leer mij vrouwen kennen: Aflevering 1',
 327                 'description': 'Leer mij vrouwen kennen: Aflevering 1',
 328                 'uploader': 'SBS Broadcasting',
 329             },
 330             'skip': 'Restricted to Netherlands',
 331             'params': {
 332                 'skip_download': True,  # m3u8 download
 333             },
 334         },
 335         # ooyala video
 336         {
 337             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
 338             'md5': '166dd577b433b4d4ebfee10b0824d8ff',
 339             'info_dict': {
 340                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
 341                 'ext': 'mp4',
 342                 'title': '2cc213299525360.mov',  # that's what we get
 343                 'duration': 238.231,
 344             },
 345             'add_ie': ['Ooyala'],
 346         },
 347         {
 348             # ooyala video embedded with http://player.ooyala.com/iframe.js
 349             'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
 350             'info_dict': {
 351                 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
 352                 'ext': 'mp4',
 353                 'title': '"Steve Jobs: Man in the Machine" trailer',
 354                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
 355                 'duration': 135.427,
 356             },
 357             'params': {
 358                 'skip_download': True,
 359             },
 360         },
 361         # multiple ooyala embeds on SBN network websites
 362         {
 363             'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
 364             'info_dict': {
 365                 'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
 366                 'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
 367             },
 368             'playlist_mincount': 3,
 369             'params': {
 370                 'skip_download': True,
 371             },
 372             'add_ie': ['Ooyala'],
 373         },
 374         # embed.ly video
 375         {
 376             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
 377             'info_dict': {
 378                 'id': '9ODmcdjQcHQ',
 379                 'ext': 'mp4',
 380                 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
 381                 'upload_date': '20140225',
 382                 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
 383                 'uploader': 'Tested',
 384                 'uploader_id': 'testedcom',
 385             },
 386             # No need to test YoutubeIE here
 387             'params': {
 388                 'skip_download': True,
 389             },
 390         },
 391         # funnyordie embed
 392         {
 393             'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
 394             'info_dict': {
 395                 'id': '18e820ec3f',
 396                 'ext': 'mp4',
 397                 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
 398                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
 399             },
 400         },
 401         # RUTV embed
 402         {
 403             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
 404             'info_dict': {
 405                 'id': '776940',
 406                 'ext': 'mp4',
 407                 'title': 'Охотское море стало целиком российским',
 408                 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
 409             },
 410             'params': {
 411                 # m3u8 download
 412                 'skip_download': True,
 413             },
 414         },
 415         # TVC embed
 416         {
 417             'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
 418             'info_dict': {
 419                 'id': '55304',
 420                 'ext': 'mp4',
 421                 'title': 'Дошкольное воспитание',
 422             },
 423         },
 424         # SportBox embed
 425         {
 426             'url': 'http://www.vestifinance.ru/articles/25753',
 427             'info_dict': {
 428                 'id': '25753',
 429                 'title': 'Вести Экономика ― Прямые трансляции с Форума-выставки "Госзаказ-2013"',
 430             },
 431             'playlist': [{
 432                 'info_dict': {
 433                     'id': '370908',
 434                     'title': 'Госзаказ. День 3',
 435                     'ext': 'mp4',
 436                 }
 437             }, {
 438                 'info_dict': {
 439                     'id': '370905',
 440                     'title': 'Госзаказ. День 2',
 441                     'ext': 'mp4',
 442                 }
 443             }, {
 444                 'info_dict': {
 445                     'id': '370902',
 446                     'title': 'Госзаказ. День 1',
 447                     'ext': 'mp4',
 448                 }
 449             }],
 450             'params': {
 451                 # m3u8 download
 452                 'skip_download': True,
 453             },
 454         },
 455         # Myvi.ru embed
 456         {
 457             'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
 458             'info_dict': {
 459                 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
 460                 'ext': 'mp4',
 461                 'title': 'Ужастики, русский трейлер (2015)',
 462                 'thumbnail': 're:^https?://.*\.jpg$',
 463                 'duration': 153,
 464             }
 465         },
 466         # XHamster embed
 467         {
 468             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
 469             'info_dict': {
 470                 'id': 'showthread',
 471                 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
 472             },
 473             'playlist_mincount': 7,
 474         },
 475         # Embedded TED video
 476         {
 477             'url': 'http://en.support.wordpress.com/videos/ted-talks/',
 478             'md5': '65fdff94098e4a607385a60c5177c638',
 479             'info_dict': {
 480                 'id': '1969',
 481                 'ext': 'mp4',
 482                 'title': 'Hidden miracles of the natural world',
 483                 'uploader': 'Louie Schwartzberg',
 484                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
 485             }
 486         },
 487         # Embeded Ustream video
 488         {
 489             'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
 490             'md5': '27b99cdb639c9b12a79bca876a073417',
 491             'info_dict': {
 492                 'id': '45734260',
 493                 'ext': 'flv',
 494                 'uploader': 'AU SPA:  The NSA and Privacy',
 495                 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
 496             }
 497         },
 498         # nowvideo embed hidden behind percent encoding
 499         {
 500             'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
 501             'md5': '2baf4ddd70f697d94b1c18cf796d5107',
 502             'info_dict': {
 503                 'id': '06e53103ca9aa',
 504                 'ext': 'flv',
 505                 'title': 'Macross Episode 001  Watch Macross Episode 001 onl',
 506                 'description': 'No description',
 507             },
 508         },
 509         # arte embed
 510         {
 511             'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
 512             'md5': '7653032cbb25bf6c80d80f217055fa43',
 513             'info_dict': {
 514                 'id': '048195-004_PLUS7-F',
 515                 'ext': 'flv',
 516                 'title': 'X:enius',
 517                 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
 518                 'upload_date': '20140320',
 519             },
 520             'params': {
 521                 'skip_download': 'Requires rtmpdump'
 522             }
 523         },
 524         # francetv embed
 525         {
 526             'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
 527             'info_dict': {
 528                 'id': 'EV_30231',
 529                 'ext': 'mp4',
 530                 'title': 'Alcaline, le concert avec Calogero',
 531                 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
 532                 'upload_date': '20150226',
 533                 'timestamp': 1424989860,
 534                 'duration': 5400,
 535             },
 536             'params': {
 537                 # m3u8 downloads
 538                 'skip_download': True,
 539             },
 540             'expected_warnings': [
 541                 'Forbidden'
 542             ]
 543         },
 544         # Condé Nast embed
 545         {
 546             'url': 'http://www.wired.com/2014/04/honda-asimo/',
 547             'md5': 'ba0dfe966fa007657bd1443ee672db0f',
 548             'info_dict': {
 549                 'id': '53501be369702d3275860000',
 550                 'ext': 'mp4',
 551                 'title': 'Honda’s  New Asimo Robot Is More Human Than Ever',
 552             }
 553         },
 554         # Dailymotion embed
 555         {
 556             'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
 557             'md5': '441aeeb82eb72c422c7f14ec533999cd',
 558             'info_dict': {
 559                 'id': 'k2mm4bCdJ6CQ2i7c8o2',
 560                 'ext': 'mp4',
 561                 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
 562                 'uploader': 'Spi0n',
 563             },
 564             'add_ie': ['Dailymotion'],
 565         },
 566         # YouTube embed
 567         {
 568             'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
 569             'info_dict': {
 570                 'id': 'FXRb4ykk4S0',
 571                 'ext': 'mp4',
 572                 'title': 'The NBL Auction 2014',
 573                 'uploader': 'BADMINTON England',
 574                 'uploader_id': 'BADMINTONEvents',
 575                 'upload_date': '20140603',
 576                 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
 577             },
 578             'add_ie': ['Youtube'],
 579             'params': {
 580                 'skip_download': True,
 581             }
 582         },
 583         # MTVSercices embed
 584         {
 585             'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
 586             'md5': '35727f82f58c76d996fc188f9755b0d5',
 587             'info_dict': {
 588                 'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
 589                 'ext': 'mp4',
 590                 'title': 'Review',
 591                 'description': 'Mario\'s life in the fast lane has never looked so good.',
 592             },
 593         },
 594         # YouTube embed via <data-embed-url="">
 595         {
 596             'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
 597             'info_dict': {
 598                 'id': '4vAffPZIT44',
 599                 'ext': 'mp4',
 600                 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
 601                 'uploader': 'Gameloft',
 602                 'uploader_id': 'gameloft',
 603                 'upload_date': '20140828',
 604                 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
 605             },
 606             'params': {
 607                 'skip_download': True,
 608             }
 609         },
 610         # Camtasia studio
 611         {
 612             'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
 613             'playlist': [{
 614                 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
 615                 'info_dict': {
 616                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 617                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
 618                     'ext': 'flv',
 619                     'duration': 2235.90,
 620                 }
 621             }, {
 622                 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
 623                 'info_dict': {
 624                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
 625                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
 626                     'ext': 'flv',
 627                     'duration': 2235.93,
 628                 }
 629             }],
 630             'info_dict': {
 631                 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 632             }
 633         },
 634         # Flowplayer
 635         {
 636             'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
 637             'md5': '9d65602bf31c6e20014319c7d07fba27',
 638             'info_dict': {
 639                 'id': '5123ea6d5e5a7',
 640                 'ext': 'mp4',
 641                 'age_limit': 18,
 642                 'uploader': 'www.handjobhub.com',
 643                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
 644             }
 645         },
 646         # Multiple brightcove videos
 647         # https://github.com/rg3/youtube-dl/issues/2283
 648         {
 649             'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
 650             'info_dict': {
 651                 'id': 'always-never',
 652                 'title': 'Always / Never - The New Yorker',
 653             },
 654             'playlist_count': 3,
 655             'params': {
 656                 'extract_flat': False,
 657                 'skip_download': True,
 658             }
 659         },
 660         # MLB embed
 661         {
 662             'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
 663             'md5': '96f09a37e44da40dd083e12d9a683327',
 664             'info_dict': {
 665                 'id': '33322633',
 666                 'ext': 'mp4',
 667                 'title': 'Ump changes call to ball',
 668                 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
 669                 'duration': 48,
 670                 'timestamp': 1401537900,
 671                 'upload_date': '20140531',
 672                 'thumbnail': 're:^https?://.*\.jpg$',
 673             },
 674         },
 675         # Wistia embed
 676         {
 677             'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
 678             'md5': '8788b683c777a5cf25621eaf286d0c23',
 679             'info_dict': {
 680                 'id': '1cfaf6b7ea',
 681                 'ext': 'mov',
 682                 'title': 'md5:51364a8d3d009997ba99656004b5e20d',
 683                 'duration': 643.0,
 684                 'filesize': 182808282,
 685                 'uploader': 'education-portal.com',
 686             },
 687         },
 688         {
 689             'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
 690             'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
 691             'info_dict': {
 692                 'id': 'uxjb0lwrcz',
 693                 'ext': 'mp4',
 694                 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
 695                 'duration': 1715.0,
 696                 'uploader': 'thoughtworks.wistia.com',
 697             },
 698         },
 699         # Soundcloud embed
 700         {
 701             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
 702             'info_dict': {
 703                 'id': '174391317',
 704                 'ext': 'mp3',
 705                 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
 706                 'uploader': 'Sophos Security',
 707                 'title': 'Chet Chat 171 - Oct 29, 2014',
 708                 'upload_date': '20141029',
 709             }
 710         },
 711         # Livestream embed
 712         {
 713             'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
 714             'info_dict': {
 715                 'id': '67864563',
 716                 'ext': 'flv',
 717                 'upload_date': '20141112',
 718                 'title': 'Rosetta #CometLanding webcast HL 10',
 719             }
 720         },
 721         # LazyYT
 722         {
 723             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
 724             'info_dict': {
 725                 'id': '1986',
 726                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
 727             },
 728             'playlist_mincount': 2,
 729         },
 730         # Cinchcast embed
 731         {
 732             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
 733             'info_dict': {
 734                 'id': '7141703',
 735                 'ext': 'mp3',
 736                 'upload_date': '20141126',
 737                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
 738             }
 739         },
 740         # Cinerama player
 741         {
 742             'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
 743             'info_dict': {
 744                 'id': '730m_DandD_1901_512k',
 745                 'ext': 'mp4',
 746                 'uploader': 'www.abc.net.au',
 747                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
 748             }
 749         },
 750         # embedded viddler video
 751         {
 752             'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
 753             'info_dict': {
 754                 'id': '4d03aad9',
 755                 'ext': 'mp4',
 756                 'uploader': 'deadspin',
 757                 'title': 'WALL-TO-GORTAT',
 758                 'timestamp': 1422285291,
 759                 'upload_date': '20150126',
 760             },
 761             'add_ie': ['Viddler'],
 762         },
 763         # Libsyn embed
 764         {
 765             'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
 766             'info_dict': {
 767                 'id': '3377616',
 768                 'ext': 'mp3',
 769                 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
 770                 'description': 'md5:601cb790edd05908957dae8aaa866465',
 771                 'upload_date': '20150220',
 772             },
 773         },
 774         # jwplayer YouTube
 775         {
 776             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
 777             'info_dict': {
 778                 'id': 'Mrj4DVp2zeA',
 779                 'ext': 'mp4',
 780                 'upload_date': '20150212',
 781                 'uploader': 'The National Archives UK',
 782                 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
 783                 'uploader_id': 'NationalArchives08',
 784                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
 785             },
 786         },
 787         # rtl.nl embed
 788         {
 789             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
 790             'playlist_mincount': 5,
 791             'info_dict': {
 792                 'id': 'aanslagen-kopenhagen',
 793                 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
 794             }
 795         },
 796         # Zapiks embed
 797         {
 798             'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
 799             'info_dict': {
 800                 'id': '118046',
 801                 'ext': 'mp4',
 802                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
 803             }
 804         },
 805         # Kaltura embed
 806         {
 807             'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
 808             'info_dict': {
 809                 'id': '1_eergr3h1',
 810                 'ext': 'mp4',
 811                 'upload_date': '20150226',
 812                 'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
 813                 'timestamp': int,
 814                 'title': 'John Carlson Postgame 2/25/15',
 815             },
 816         },
 817         # Kaltura embed (different embed code)
 818         {
 819             'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
 820             'info_dict': {
 821                 'id': '1_a52wc67y',
 822                 'ext': 'flv',
 823                 'upload_date': '20150127',
 824                 'uploader_id': 'PremierMedia',
 825                 'timestamp': int,
 826                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
 827             },
 828         },
 829         # Kaltura embed protected with referrer
 830         {
 831             'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
 832             'info_dict': {
 833                 'id': '1_g4fbemnq',
 834                 'ext': 'mp4',
 835                 'title': 'Violetta - Achter De Schermen - Ruggero',
 836                 'description': 'Achter de schermen met Ruggero',
 837                 'timestamp': 1435133761,
 838                 'upload_date': '20150624',
 839                 'uploader_id': 'echojecka',
 840             },
 841         },
 842         # Eagle.Platform embed (generic URL)
 843         {
 844             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
 845             'info_dict': {
 846                 'id': '227304',
 847                 'ext': 'mp4',
 848                 'title': 'Навальный вышел на свободу',
 849                 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
 850                 'thumbnail': 're:^https?://.*\.jpg$',
 851                 'duration': 87,
 852                 'view_count': int,
 853                 'age_limit': 0,
 854             },
 855         },
 856         # ClipYou (Eagle.Platform) embed (custom URL)
 857         {
 858             'url': 'http://muz-tv.ru/play/7129/',
 859             'info_dict': {
 860                 'id': '12820',
 861                 'ext': 'mp4',
 862                 'title': "'O Sole Mio",
 863                 'thumbnail': 're:^https?://.*\.jpg$',
 864                 'duration': 216,
 865                 'view_count': int,
 866             },
 867         },
 868         # Pladform embed
 869         {
 870             'url': 'http://muz-tv.ru/kinozal/view/7400/',
 871             'info_dict': {
 872                 'id': '100183293',
 873                 'ext': 'mp4',
 874                 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
 875                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
 876                 'thumbnail': 're:^https?://.*\.jpg$',
 877                 'duration': 694,
 878                 'age_limit': 0,
 879             },
 880         },
 881         # Playwire embed
 882         {
 883             'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
 884             'info_dict': {
 885                 'id': '3519514',
 886                 'ext': 'mp4',
 887                 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
 888                 'thumbnail': 're:^https?://.*\.png$',
 889                 'duration': 45.115,
 890             },
 891         },
 892         # 5min embed
 893         {
 894             'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
 895             'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
 896             'info_dict': {
 897                 'id': '518726732',
 898                 'ext': 'mp4',
 899                 'title': 'Facebook Creates "On This Day" | Crunch Report',
 900             },
 901         },
 902         # SVT embed
 903         {
 904             'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
 905             'info_dict': {
 906                 'id': '2900353',
 907                 'ext': 'flv',
 908                 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
 909                 'duration': 27,
 910                 'age_limit': 0,
 911             },
 912         },
 913         # Crooks and Liars embed
 914         {
 915             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
 916             'info_dict': {
 917                 'id': '8RUoRhRi',
 918                 'ext': 'mp4',
 919                 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
 920                 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
 921                 'timestamp': 1428207000,
 922                 'upload_date': '20150405',
 923                 'uploader': 'Heather',
 924             },
 925         },
 926         # Crooks and Liars external embed
 927         {
 928             'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
 929             'info_dict': {
 930                 'id': 'MTE3MjUtMzQ2MzA',
 931                 'ext': 'mp4',
 932                 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
 933                 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
 934                 'timestamp': 1265032391,
 935                 'upload_date': '20100201',
 936                 'uploader': 'Heather',
 937             },
 938         },
 939         # NBC Sports vplayer embed
 940         {
 941             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
 942             'info_dict': {
 943                 'id': 'ln7x1qSThw4k',
 944                 'ext': 'flv',
 945                 'title': "PFT Live: New leader in the 'new-look' defense",
 946                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
 947             },
 948         },
 949         # UDN embed
 950         {
 951             'url': 'http://www.udn.com/news/story/7314/822787',
 952             'md5': 'fd2060e988c326991037b9aff9df21a6',
 953             'info_dict': {
 954                 'id': '300346',
 955                 'ext': 'mp4',
 956                 'title': '中一中男師變性 全校師生力挺',
 957                 'thumbnail': 're:^https?://.*\.jpg$',
 958             }
 959         },
 960         # Ooyala embed
 961         {
 962             'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
 963             'info_dict': {
 964                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
 965                 'ext': 'mp4',
 966                 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
 967                 'title': 'This is what separates the Excel masters from the wannabes',
 968                 'duration': 191.933,
 969             },
 970             'params': {
 971                 # m3u8 downloads
 972                 'skip_download': True,
 973             }
 974         },
 975         # Contains a SMIL manifest
 976         {
 977             'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
 978             'info_dict': {
 979                 'id': 'file',
 980                 'ext': 'flv',
 981                 'title': '+ Football: Lottery Champions League Europe',
 982                 'uploader': 'www.telewebion.com',
 983             },
 984             'params': {
 985                 # rtmpe downloads
 986                 'skip_download': True,
 987             }
 988         },
 989         # Brightcove URL in single quotes
 990         {
 991             'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
 992             'md5': '4ae374f1f8b91c889c4b9203c8c752af',
 993             'info_dict': {
 994                 'id': '4255764656001',
 995                 'ext': 'mp4',
 996                 'title': 'SN Presents: Russell Martin, World Citizen',
 997                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
 998                 'uploader': 'Rogers Sportsnet',
 999             },
1000         },
1001         # Dailymotion Cloud video
1002         {
1003             'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
1004             'md5': '49444254273501a64675a7e68c502681',
1005             'info_dict': {
1006                 'id': '5585de919473990de4bee11b',
1007                 'ext': 'mp4',
1008                 'title': 'Le débat',
1009                 'thumbnail': 're:^https?://.*\.jpe?g$',
1010             }
1011         },
1012         # OnionStudios embed
1013         {
1014             'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1015             'info_dict': {
1016                 'id': '2855',
1017                 'ext': 'mp4',
1018                 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1019                 'thumbnail': 're:^https?://.*\.jpe?g$',
1020                 'uploader': 'ClickHole',
1021                 'uploader_id': 'clickhole',
1022             }
1023         },
1024         # SnagFilms embed
1025         {
1026             'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1027             'info_dict': {
1028                 'id': '74849a00-85a9-11e1-9660-123139220831',
1029                 'ext': 'mp4',
1030                 'title': '#whilewewatch',
1031             }
1032         },
1033         # AdobeTVVideo embed
1034         {
1035             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1036             'md5': '43662b577c018ad707a63766462b1e87',
1037             'info_dict': {
1038                 'id': '2456',
1039                 'ext': 'mp4',
1040                 'title': 'New experience with Acrobat DC',
1041                 'description': 'New experience with Acrobat DC',
1042                 'duration': 248.667,
1043             },
1044         },
1045         # ScreenwaveMedia embed
1046         {
1047             'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
1048             'md5': '24ace5baba0d35d55c6810b51f34e9e0',
1049             'info_dict': {
1050                 'id': 'cinemasnob-55d26273809dd',
1051                 'ext': 'mp4',
1052                 'title': 'cinemasnob',
1053             },
1054         },
1055         # BrightcoveInPageEmbed embed
1056         {
1057             'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1058             'info_dict': {
1059                 'id': '4238694884001',
1060                 'ext': 'flv',
1061                 'title': 'Tabletop: Dread, Last Thoughts',
1062                 'description': 'Tabletop: Dread, Last Thoughts',
1063                 'duration': 51690,
1064             },
1065         },
1066         # JWPlayer with M3U8
1067         {
1068             'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
1069             'info_dict': {
1070                 'id': 'playlist',
1071                 'ext': 'mp4',
1072                 'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
1073                 'uploader': 'ren.tv',
1074             },
1075             'params': {
1076                 # m3u8 downloads
1077                 'skip_download': True,
1078             }
1079         }
1080     ]
1081
1082     def report_following_redirect(self, new_url):
1083         """Report information extraction."""
1084         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
1085
1086     def _extract_rss(self, url, video_id, doc):
1087         playlist_title = doc.find('./channel/title').text
1088         playlist_desc_el = doc.find('./channel/description')
1089         playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
1090
1091         entries = []
1092         for it in doc.findall('./channel/item'):
1093             next_url = xpath_text(it, 'link', fatal=False)
1094             if not next_url:
1095                 enclosure_nodes = it.findall('./enclosure')
1096                 for e in enclosure_nodes:
1097                     next_url = e.attrib.get('url')
1098                     if next_url:
1099                         break
1100
1101             if not next_url:
1102                 continue
1103
1104             entries.append({
1105                 '_type': 'url',
1106                 'url': next_url,
1107                 'title': it.find('title').text,
1108             })
1109
1110         return {
1111             '_type': 'playlist',
1112             'id': url,
1113             'title': playlist_title,
1114             'description': playlist_desc,
1115             'entries': entries,
1116         }
1117
1118     def _extract_camtasia(self, url, video_id, webpage):
1119         """ Returns None if no camtasia video can be found. """
1120
1121         camtasia_cfg = self._search_regex(
1122             r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
1123             webpage, 'camtasia configuration file', default=None)
1124         if camtasia_cfg is None:
1125             return None
1126
1127         title = self._html_search_meta('DC.title', webpage, fatal=True)
1128
1129         camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
1130         camtasia_cfg = self._download_xml(
1131             camtasia_url, video_id,
1132             note='Downloading camtasia configuration',
1133             errnote='Failed to download camtasia configuration')
1134         fileset_node = camtasia_cfg.find('./playlist/array/fileset')
1135
1136         entries = []
1137         for n in fileset_node.getchildren():
1138             url_n = n.find('./uri')
1139             if url_n is None:
1140                 continue
1141
1142             entries.append({
1143                 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
1144                 'title': '%s - %s' % (title, n.tag),
1145                 'url': compat_urlparse.urljoin(url, url_n.text),
1146                 'duration': float_or_none(n.find('./duration').text),
1147             })
1148
1149         return {
1150             '_type': 'playlist',
1151             'entries': entries,
1152             'title': title,
1153         }
1154
1155     def _real_extract(self, url):
1156         if url.startswith('//'):
1157             return {
1158                 '_type': 'url',
1159                 'url': self.http_scheme() + url,
1160             }
1161
1162         parsed_url = compat_urlparse.urlparse(url)
1163         if not parsed_url.scheme:
1164             default_search = self._downloader.params.get('default_search')
1165             if default_search is None:
1166                 default_search = 'fixup_error'
1167
1168             if default_search in ('auto', 'auto_warning', 'fixup_error'):
1169                 if '/' in url:
1170                     self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
1171                     return self.url_result('http://' + url)
1172                 elif default_search != 'fixup_error':
1173                     if default_search == 'auto_warning':
1174                         if re.match(r'^(?:url|URL)$', url):
1175                             raise ExtractorError(
1176                                 'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
1177                                 expected=True)
1178                         else:
1179                             self._downloader.report_warning(
1180                                 'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
1181                     return self.url_result('ytsearch:' + url)
1182
1183             if default_search in ('error', 'fixup_error'):
1184                 raise ExtractorError(
1185                     '%r is not a valid URL. '
1186                     'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
1187                     % (url, url), expected=True)
1188             else:
1189                 if ':' not in default_search:
1190                     default_search += ':'
1191                 return self.url_result(default_search + url)
1192
1193         url, smuggled_data = unsmuggle_url(url)
1194         force_videoid = None
1195         is_intentional = smuggled_data and smuggled_data.get('to_generic')
1196         if smuggled_data and 'force_videoid' in smuggled_data:
1197             force_videoid = smuggled_data['force_videoid']
1198             video_id = force_videoid
1199         else:
1200             video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
1201
1202         self.to_screen('%s: Requesting header' % video_id)
1203
1204         head_req = HEADRequest(url)
1205         head_response = self._request_webpage(
1206             head_req, video_id,
1207             note=False, errnote='Could not send HEAD request to %s' % url,
1208             fatal=False)
1209
1210         if head_response is not False:
1211             # Check for redirect
1212             new_url = head_response.geturl()
1213             if url != new_url:
1214                 self.report_following_redirect(new_url)
1215                 if force_videoid:
1216                     new_url = smuggle_url(
1217                         new_url, {'force_videoid': force_videoid})
1218                 return self.url_result(new_url)
1219
1220         full_response = None
1221         if head_response is False:
1222             request = sanitized_Request(url)
1223             request.add_header('Accept-Encoding', '*')
1224             full_response = self._request_webpage(request, video_id)
1225             head_response = full_response
1226
1227         # Check for direct link to a video
1228         content_type = head_response.headers.get('Content-Type', '')
1229         m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
1230         if m:
1231             upload_date = unified_strdate(
1232                 head_response.headers.get('Last-Modified'))
1233             return {
1234                 'id': video_id,
1235                 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
1236                 'direct': True,
1237                 'formats': [{
1238                     'format_id': m.group('format_id'),
1239                     'url': url,
1240                     'vcodec': 'none' if m.group('type') == 'audio' else None
1241                 }],
1242                 'upload_date': upload_date,
1243             }
1244
1245         if not self._downloader.params.get('test', False) and not is_intentional:
1246             force = self._downloader.params.get('force_generic_extractor', False)
1247             self._downloader.report_warning(
1248                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
1249
1250         if not full_response:
1251             request = sanitized_Request(url)
1252             # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
1253             # making it impossible to download only chunk of the file (yet we need only 512kB to
1254             # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
1255             # that will always result in downloading the whole file that is not desirable.
1256             # Therefore for extraction pass we have to override Accept-Encoding to any in order
1257             # to accept raw bytes and being able to download only a chunk.
1258             # It may probably better to solve this by checking Content-Type for application/octet-stream
1259             # after HEAD request finishes, but not sure if we can rely on this.
1260             request.add_header('Accept-Encoding', '*')
1261             full_response = self._request_webpage(request, video_id)
1262
1263         # Maybe it's a direct link to a video?
1264         # Be careful not to download the whole thing!
1265         first_bytes = full_response.read(512)
1266         if not is_html(first_bytes):
1267             self._downloader.report_warning(
1268                 'URL could be a direct video link, returning it as such.')
1269             upload_date = unified_strdate(
1270                 head_response.headers.get('Last-Modified'))
1271             return {
1272                 'id': video_id,
1273                 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
1274                 'direct': True,
1275                 'url': url,
1276                 'upload_date': upload_date,
1277             }
1278
1279         webpage = self._webpage_read_content(
1280             full_response, url, video_id, prefix=first_bytes)
1281
1282         self.report_extraction(video_id)
1283
1284         # Is it an RSS feed, a SMIL file or a XSPF playlist?
1285         try:
1286             doc = compat_etree_fromstring(webpage.encode('utf-8'))
1287             if doc.tag == 'rss':
1288                 return self._extract_rss(url, video_id, doc)
1289             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
1290                 return self._parse_smil(doc, url, video_id)
1291             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
1292                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
1293         except compat_xml_parse_error:
1294             pass
1295
1296         # Is it a Camtasia project?
1297         camtasia_res = self._extract_camtasia(url, video_id, webpage)
1298         if camtasia_res is not None:
1299             return camtasia_res
1300
1301         # Sometimes embedded video player is hidden behind percent encoding
1302         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1303         # Unescaping the whole page allows to handle those cases in a generic way
1304         webpage = compat_urllib_parse_unquote(webpage)
1305
1306         # it's tempting to parse this further, but you would
1307         # have to take into account all the variations like
1308         #   Video Title - Site Name
1309         #   Site Name | Video Title
1310         #   Video Title - Tagline | Site Name
1311         # and so on and so forth; it's just not practical
1312         video_title = self._html_search_regex(
1313             r'(?s)<title>(.*?)</title>', webpage, 'video title',
1314             default='video')
1315
1316         # Try to detect age limit automatically
1317         age_limit = self._rta_search(webpage)
1318         # And then there are the jokers who advertise that they use RTA,
1319         # but actually don't.
1320         AGE_LIMIT_MARKERS = [
1321             r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1322         ]
1323         if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
1324             age_limit = 18
1325
1326         # video uploader is domain name
1327         video_uploader = self._search_regex(
1328             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
1329
1330         # Helper method
1331         def _playlist_from_matches(matches, getter=None, ie=None):
1332             urlrs = orderedSet(
1333                 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
1334                 for m in matches)
1335             return self.playlist_result(
1336                 urlrs, playlist_id=video_id, playlist_title=video_title)
1337
1338         # Look for Brightcove Legacy Studio embeds
1339         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
1340         if bc_urls:
1341             self.to_screen('Brightcove video detected.')
1342             entries = [{
1343                 '_type': 'url',
1344                 'url': smuggle_url(bc_url, {'Referer': url}),
1345                 'ie_key': 'BrightcoveLegacy'
1346             } for bc_url in bc_urls]
1347
1348             return {
1349                 '_type': 'playlist',
1350                 'title': video_title,
1351                 'id': video_id,
1352                 'entries': entries,
1353             }
1354
1355         # Look for Brightcove New Studio embeds
1356         bc_urls = BrightcoveNewIE._extract_urls(webpage)
1357         if bc_urls:
1358             return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
1359
1360         # Look for embedded rtl.nl player
1361         matches = re.findall(
1362             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
1363             webpage)
1364         if matches:
1365             return _playlist_from_matches(matches, ie='RtlNl')
1366
1367         vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
1368         if vimeo_url is not None:
1369             return self.url_result(vimeo_url)
1370
1371         vid_me_embed_url = self._search_regex(
1372             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
1373             webpage, 'vid.me embed', default=None)
1374         if vid_me_embed_url is not None:
1375             return self.url_result(vid_me_embed_url, 'Vidme')
1376
1377         # Look for embedded YouTube player
1378         matches = re.findall(r'''(?x)
1379             (?:
1380                 <iframe[^>]+?src=|
1381                 data-video-url=|
1382                 <embed[^>]+?src=|
1383                 embedSWF\(?:\s*|
1384                 new\s+SWFObject\(
1385             )
1386             (["\'])
1387                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1388                 (?:embed|v|p)/.+?)
1389             \1''', webpage)
1390         if matches:
1391             return _playlist_from_matches(
1392                 matches, lambda m: unescapeHTML(m[1]))
1393
1394         # Look for lazyYT YouTube embed
1395         matches = re.findall(
1396             r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1397         if matches:
1398             return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1399
1400         # Look for embedded Dailymotion player
1401         matches = re.findall(
1402             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
1403         if matches:
1404             return _playlist_from_matches(
1405                 matches, lambda m: unescapeHTML(m[1]))
1406
1407         # Look for embedded Dailymotion playlist player (#3822)
1408         m = re.search(
1409             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1410         if m:
1411             playlists = re.findall(
1412                 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1413             if playlists:
1414                 return _playlist_from_matches(
1415                     playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
1416
1417         # Look for embedded Wistia player
1418         match = re.search(
1419             r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1420         if match:
1421             embed_url = self._proto_relative_url(
1422                 unescapeHTML(match.group('url')))
1423             return {
1424                 '_type': 'url_transparent',
1425                 'url': embed_url,
1426                 'ie_key': 'Wistia',
1427                 'uploader': video_uploader,
1428                 'title': video_title,
1429                 'id': video_id,
1430             }
1431
1432         match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1433         if match:
1434             return {
1435                 '_type': 'url_transparent',
1436                 'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
1437                 'ie_key': 'Wistia',
1438                 'uploader': video_uploader,
1439                 'title': video_title,
1440                 'id': match.group('id')
1441             }
1442
1443         # Look for embedded blip.tv player
1444         bliptv_url = BlipTVIE._extract_url(webpage)
1445         if bliptv_url:
1446             return self.url_result(bliptv_url, 'BlipTV')
1447
1448         # Look for SVT player
1449         svt_url = SVTIE._extract_url(webpage)
1450         if svt_url:
1451             return self.url_result(svt_url, 'SVT')
1452
1453         # Look for embedded condenast player
1454         matches = re.findall(
1455             r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1456             webpage)
1457         if matches:
1458             return {
1459                 '_type': 'playlist',
1460                 'entries': [{
1461                     '_type': 'url',
1462                     'ie_key': 'CondeNast',
1463                     'url': ma,
1464                 } for ma in matches],
1465                 'title': video_title,
1466                 'id': video_id,
1467             }
1468
1469         # Look for Bandcamp pages with custom domain
1470         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1471         if mobj is not None:
1472             burl = unescapeHTML(mobj.group(1))
1473             # Don't set the extractor because it can be a track url or an album
1474             return self.url_result(burl)
1475
1476         # Look for embedded Vevo player
1477         mobj = re.search(
1478             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1479         if mobj is not None:
1480             return self.url_result(mobj.group('url'))
1481
1482         # Look for embedded Viddler player
1483         mobj = re.search(
1484             r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1485             webpage)
1486         if mobj is not None:
1487             return self.url_result(mobj.group('url'))
1488
1489         # Look for NYTimes player
1490         mobj = re.search(
1491             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1492             webpage)
1493         if mobj is not None:
1494             return self.url_result(mobj.group('url'))
1495
1496         # Look for Libsyn player
1497         mobj = re.search(
1498             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1499         if mobj is not None:
1500             return self.url_result(mobj.group('url'))
1501
1502         # Look for Ooyala videos
1503         mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1504                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1505                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1506                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1507         if mobj is not None:
1508             return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
1509
1510         # Look for multiple Ooyala embeds on SBN network websites
1511         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1512         if mobj is not None:
1513             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1514             if embeds:
1515                 return _playlist_from_matches(
1516                     embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
1517
1518         # Look for Aparat videos
1519         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1520         if mobj is not None:
1521             return self.url_result(mobj.group(1), 'Aparat')
1522
1523         # Look for MPORA videos
1524         mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1525         if mobj is not None:
1526             return self.url_result(mobj.group(1), 'Mpora')
1527
1528         # Look for embedded NovaMov-based player
1529         mobj = re.search(
1530             r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
1531                     (?P<url>http://(?:(?:embed|www)\.)?
1532                         (?:novamov\.com|
1533                            nowvideo\.(?:ch|sx|eu|at|ag|co)|
1534                            videoweed\.(?:es|com)|
1535                            movshare\.(?:net|sx|ag)|
1536                            divxstage\.(?:eu|net|ch|co|at|ag))
1537                         /embed\.php.+?)\1''', webpage)
1538         if mobj is not None:
1539             return self.url_result(mobj.group('url'))
1540
1541         # Look for embedded Facebook player
1542         mobj = re.search(
1543             r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
1544         if mobj is not None:
1545             return self.url_result(mobj.group('url'), 'Facebook')
1546
1547         # Look for embedded VK player
1548         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
1549         if mobj is not None:
1550             return self.url_result(mobj.group('url'), 'VK')
1551
1552         # Look for embedded ivi player
1553         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
1554         if mobj is not None:
1555             return self.url_result(mobj.group('url'), 'Ivi')
1556
1557         # Look for embedded Huffington Post player
1558         mobj = re.search(
1559             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1560         if mobj is not None:
1561             return self.url_result(mobj.group('url'), 'HuffPost')
1562
1563         # Look for embed.ly
1564         mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
1565         if mobj is not None:
1566             return self.url_result(mobj.group('url'))
1567         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
1568         if mobj is not None:
1569             return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
1570
1571         # Look for funnyordie embed
1572         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
1573         if matches:
1574             return _playlist_from_matches(
1575                 matches, getter=unescapeHTML, ie='FunnyOrDie')
1576
1577         # Look for BBC iPlayer embed
1578         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
1579         if matches:
1580             return _playlist_from_matches(matches, ie='BBCCoUk')
1581
1582         # Look for embedded RUTV player
1583         rutv_url = RUTVIE._extract_url(webpage)
1584         if rutv_url:
1585             return self.url_result(rutv_url, 'RUTV')
1586
1587         # Look for embedded TVC player
1588         tvc_url = TVCIE._extract_url(webpage)
1589         if tvc_url:
1590             return self.url_result(tvc_url, 'TVC')
1591
1592         # Look for embedded SportBox player
1593         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
1594         if sportbox_urls:
1595             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
1596
1597         # Look for embedded PornHub player
1598         pornhub_url = PornHubIE._extract_url(webpage)
1599         if pornhub_url:
1600             return self.url_result(pornhub_url, 'PornHub')
1601
1602         # Look for embedded XHamster player
1603         xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
1604         if xhamster_urls:
1605             return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
1606
1607         # Look for embedded Tvigle player
1608         mobj = re.search(
1609             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
1610         if mobj is not None:
1611             return self.url_result(mobj.group('url'), 'Tvigle')
1612
1613         # Look for embedded TED player
1614         mobj = re.search(
1615             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
1616         if mobj is not None:
1617             return self.url_result(mobj.group('url'), 'TED')
1618
1619         # Look for embedded Ustream videos
1620         mobj = re.search(
1621             r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
1622         if mobj is not None:
1623             return self.url_result(mobj.group('url'), 'Ustream')
1624
1625         # Look for embedded arte.tv player
1626         mobj = re.search(
1627             r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
1628             webpage)
1629         if mobj is not None:
1630             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
1631
1632         # Look for embedded francetv player
1633         mobj = re.search(
1634             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
1635             webpage)
1636         if mobj is not None:
1637             return self.url_result(mobj.group('url'))
1638
1639         # Look for embedded smotri.com player
1640         smotri_url = SmotriIE._extract_url(webpage)
1641         if smotri_url:
1642             return self.url_result(smotri_url, 'Smotri')
1643
1644         # Look for embedded Myvi.ru player
1645         myvi_url = MyviIE._extract_url(webpage)
1646         if myvi_url:
1647             return self.url_result(myvi_url)
1648
1649         # Look for embeded soundcloud player
1650         mobj = re.search(
1651             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
1652             webpage)
1653         if mobj is not None:
1654             url = unescapeHTML(mobj.group('url'))
1655             return self.url_result(url)
1656
1657         # Look for embedded vulture.com player
1658         mobj = re.search(
1659             r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
1660             webpage)
1661         if mobj is not None:
1662             url = unescapeHTML(mobj.group('url'))
1663             return self.url_result(url, ie='Vulture')
1664
1665         # Look for embedded mtvservices player
1666         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
1667         if mtvservices_url:
1668             return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
1669
1670         # Look for embedded yahoo player
1671         mobj = re.search(
1672             r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
1673             webpage)
1674         if mobj is not None:
1675             return self.url_result(mobj.group('url'), 'Yahoo')
1676
1677         # Look for embedded sbs.com.au player
1678         mobj = re.search(
1679             r'''(?x)
1680             (?:
1681                 <meta\s+property="og:video"\s+content=|
1682                 <iframe[^>]+?src=
1683             )
1684             (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
1685             webpage)
1686         if mobj is not None:
1687             return self.url_result(mobj.group('url'), 'SBS')
1688
1689         # Look for embedded Cinchcast player
1690         mobj = re.search(
1691             r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
1692             webpage)
1693         if mobj is not None:
1694             return self.url_result(mobj.group('url'), 'Cinchcast')
1695
1696         mobj = re.search(
1697             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
1698             webpage)
1699         if not mobj:
1700             mobj = re.search(
1701                 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
1702                 webpage)
1703         if mobj is not None:
1704             return self.url_result(mobj.group('url'), 'MLB')
1705
1706         mobj = re.search(
1707             r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
1708             webpage)
1709         if mobj is not None:
1710             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
1711
1712         mobj = re.search(
1713             r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
1714             webpage)
1715         if mobj is not None:
1716             return self.url_result(mobj.group('url'), 'Livestream')
1717
1718         # Look for Zapiks embed
1719         mobj = re.search(
1720             r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
1721         if mobj is not None:
1722             return self.url_result(mobj.group('url'), 'Zapiks')
1723
1724         # Look for Kaltura embeds
1725         mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
1726                 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
1727         if mobj is not None:
1728             return self.url_result(smuggle_url(
1729                 'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
1730                 {'source_url': url}), 'Kaltura')
1731
1732         # Look for Eagle.Platform embeds
1733         mobj = re.search(
1734             r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
1735         if mobj is not None:
1736             return self.url_result(mobj.group('url'), 'EaglePlatform')
1737
1738         # Look for ClipYou (uses Eagle.Platform) embeds
1739         mobj = re.search(
1740             r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
1741         if mobj is not None:
1742             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
1743
1744         # Look for Pladform embeds
1745         pladform_url = PladformIE._extract_url(webpage)
1746         if pladform_url:
1747             return self.url_result(pladform_url)
1748
1749         # Look for Playwire embeds
1750         mobj = re.search(
1751             r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
1752         if mobj is not None:
1753             return self.url_result(mobj.group('url'))
1754
1755         # Look for 5min embeds
1756         mobj = re.search(
1757             r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
1758         if mobj is not None:
1759             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
1760
1761         # Look for Crooks and Liars embeds
1762         mobj = re.search(
1763             r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
1764         if mobj is not None:
1765             return self.url_result(mobj.group('url'))
1766
1767         # Look for NBC Sports VPlayer embeds
1768         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
1769         if nbc_sports_url:
1770             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
1771
1772         # Look for UDN embeds
1773         mobj = re.search(
1774             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
1775         if mobj is not None:
1776             return self.url_result(
1777                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
1778
1779         # Look for Senate ISVP iframe
1780         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
1781         if senate_isvp_url:
1782             return self.url_result(senate_isvp_url, 'SenateISVP')
1783
1784         # Look for Dailymotion Cloud videos
1785         dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
1786         if dmcloud_url:
1787             return self.url_result(dmcloud_url, 'DailymotionCloud')
1788
1789         # Look for OnionStudios embeds
1790         onionstudios_url = OnionStudiosIE._extract_url(webpage)
1791         if onionstudios_url:
1792             return self.url_result(onionstudios_url)
1793
1794         # Look for SnagFilms embeds
1795         snagfilms_url = SnagFilmsEmbedIE._extract_url(webpage)
1796         if snagfilms_url:
1797             return self.url_result(snagfilms_url)
1798
1799         # Look for ScreenwaveMedia embeds
1800         mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
1801         if mobj is not None:
1802             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
1803
1804         # Look for AdobeTVVideo embeds
1805         mobj = re.search(
1806             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
1807             webpage)
1808         if mobj is not None:
1809             return self.url_result(
1810                 self._proto_relative_url(unescapeHTML(mobj.group(1))),
1811                 'AdobeTVVideo')
1812
1813         def check_video(vurl):
1814             if YoutubeIE.suitable(vurl):
1815                 return True
1816             vpath = compat_urlparse.urlparse(vurl).path
1817             vext = determine_ext(vpath)
1818             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
1819
1820         def filter_video(urls):
1821             return list(filter(check_video, urls))
1822
1823         # Start with something easy: JW Player in SWFObject
1824         found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
1825         if not found:
1826             # Look for gorilla-vid style embedding
1827             found = filter_video(re.findall(r'''(?sx)
1828                 (?:
1829                     jw_plugins|
1830                     JWPlayerOptions|
1831                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
1832                 )
1833                 .*?
1834                 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
1835         if not found:
1836             # Broaden the search a little bit
1837             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
1838         if not found:
1839             # Broaden the findall a little bit: JWPlayer JS loader
1840             found = filter_video(re.findall(
1841                 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
1842         if not found:
1843             # Flow player
1844             found = filter_video(re.findall(r'''(?xs)
1845                 flowplayer\("[^"]+",\s*
1846                     \{[^}]+?\}\s*,
1847                     \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
1848                         ["']?url["']?\s*:\s*["']([^"']+)["']
1849             ''', webpage))
1850         if not found:
1851             # Cinerama player
1852             found = re.findall(
1853                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
1854         if not found:
1855             # Try to find twitter cards info
1856             found = filter_video(re.findall(
1857                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
1858         if not found:
1859             # We look for Open Graph info:
1860             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
1861             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
1862             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
1863             if m_video_type is not None:
1864                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
1865         if not found:
1866             # HTML5 video
1867             found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
1868         if not found:
1869             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
1870             found = re.search(
1871                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
1872                 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
1873                 webpage)
1874             if not found:
1875                 # Look also in Refresh HTTP header
1876                 refresh_header = head_response.headers.get('Refresh')
1877                 if refresh_header:
1878                     # In python 2 response HTTP headers are bytestrings
1879                     if sys.version_info < (3, 0) and isinstance(refresh_header, str):
1880                         refresh_header = refresh_header.decode('iso-8859-1')
1881                     found = re.search(REDIRECT_REGEX, refresh_header)
1882             if found:
1883                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
1884                 self.report_following_redirect(new_url)
1885                 return {
1886                     '_type': 'url',
1887                     'url': new_url,
1888                 }
1889         if not found:
1890             raise UnsupportedError(url)
1891
1892         entries = []
1893         for video_url in found:
1894             video_url = video_url.replace('\\/', '/')
1895             video_url = compat_urlparse.urljoin(url, video_url)
1896             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
1897
1898             # Sometimes, jwplayer extraction will result in a YouTube URL
1899             if YoutubeIE.suitable(video_url):
1900                 entries.append(self.url_result(video_url, 'Youtube'))
1901                 continue
1902
1903             # here's a fun little line of code for you:
1904             video_id = os.path.splitext(video_id)[0]
1905
1906             entry_info_dict = {
1907                 'id': video_id,
1908                 'uploader': video_uploader,
1909                 'title': video_title,
1910                 'age_limit': age_limit,
1911             }
1912
1913             ext = determine_ext(video_url)
1914             if ext == 'smil':
1915                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
1916             elif ext == 'xspf':
1917                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
1918             elif ext == 'm3u8':
1919                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
1920             else:
1921                 entry_info_dict['url'] = video_url
1922
1923             entries.append(entry_info_dict)
1924
1925         if len(entries) == 1:
1926             return entries[0]
1927         else:
1928             for num, e in enumerate(entries, start=1):
1929                 # 'url' results don't have a title
1930                 if e.get('title') is not None:
1931                     e['title'] = '%s (%d)' % (e['title'], num)
1932             return {
1933                 '_type': 'playlist',
1934                 'entries': entries,
1935             }