_ Git - youtube-dl/blob - youtube_dl/extractor/generic.py

   1 # encoding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import os
   6 import re
   7 import sys
   8
   9 from .common import InfoExtractor
  10 from .youtube import YoutubeIE
  11 from ..compat import (
  12     compat_etree_fromstring,
  13     compat_urllib_parse_unquote,
  14     compat_urlparse,
  15     compat_xml_parse_error,
  16 )
  17 from ..utils import (
  18     determine_ext,
  19     ExtractorError,
  20     float_or_none,
  21     HEADRequest,
  22     is_html,
  23     orderedSet,
  24     sanitized_Request,
  25     smuggle_url,
  26     unescapeHTML,
  27     unified_strdate,
  28     unsmuggle_url,
  29     UnsupportedError,
  30     url_basename,
  31     xpath_text,
  32 )
  33 from .brightcove import (
  34     BrightcoveLegacyIE,
  35     BrightcoveNewIE,
  36 )
  37 from .nbc import NBCSportsVPlayerIE
  38 from .ooyala import OoyalaIE
  39 from .rutv import RUTVIE
  40 from .tvc import TVCIE
  41 from .sportbox import SportBoxEmbedIE
  42 from .smotri import SmotriIE
  43 from .myvi import MyviIE
  44 from .condenast import CondeNastIE
  45 from .udn import UDNEmbedIE
  46 from .senateisvp import SenateISVPIE
  47 from .svt import SVTIE
  48 from .pornhub import PornHubIE
  49 from .xhamster import XHamsterEmbedIE
  50 from .tnaflix import TNAFlixNetworkEmbedIE
  51 from .vimeo import VimeoIE
  52 from .dailymotion import DailymotionCloudIE
  53 from .onionstudios import OnionStudiosIE
  54 from .snagfilms import SnagFilmsEmbedIE
  55 from .screenwavemedia import ScreenwaveMediaIE
  56 from .mtv import MTVServicesEmbeddedIE
  57 from .pladform import PladformIE
  58 from .videomore import VideomoreIE
  59 from .googledrive import GoogleDriveIE
  60 from .jwplatform import JWPlatformIE
  61 from .digiteka import DigitekaIE
  62 from .instagram import InstagramIE
  63 from .liveleak import LiveLeakIE
  64
  65
  66 class GenericIE(InfoExtractor):
  67     IE_DESC = 'Generic downloader that works on some sites'
  68     _VALID_URL = r'.*'
  69     IE_NAME = 'generic'
  70     _TESTS = [
  71         # Direct link to a video
  72         {
  73             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
  74             'md5': '67d406c2bcb6af27fa886f31aa934bbe',
  75             'info_dict': {
  76                 'id': 'trailer',
  77                 'ext': 'mp4',
  78                 'title': 'trailer',
  79                 'upload_date': '20100513',
  80             }
  81         },
  82         # Direct link to media delivered compressed (until Accept-Encoding is *)
  83         {
  84             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
  85             'md5': '128c42e68b13950268b648275386fc74',
  86             'info_dict': {
  87                 'id': 'FictionJunction-Parallel_Hearts',
  88                 'ext': 'flac',
  89                 'title': 'FictionJunction-Parallel_Hearts',
  90                 'upload_date': '20140522',
  91             },
  92             'expected_warnings': [
  93                 'URL could be a direct video link, returning it as such.'
  94             ]
  95         },
  96         # Direct download with broken HEAD
  97         {
  98             'url': 'http://ai-radio.org:8000/radio.opus',
  99             'info_dict': {
 100                 'id': 'radio',
 101                 'ext': 'opus',
 102                 'title': 'radio',
 103             },
 104             'params': {
 105                 'skip_download': True,  # infinite live stream
 106             },
 107             'expected_warnings': [
 108                 r'501.*Not Implemented',
 109                 r'400.*Bad Request',
 110             ],
 111         },
 112         # Direct link with incorrect MIME type
 113         {
 114             'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 115             'md5': '4ccbebe5f36706d85221f204d7eb5913',
 116             'info_dict': {
 117                 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 118                 'id': '5_Lennart_Poettering_-_Systemd',
 119                 'ext': 'webm',
 120                 'title': '5_Lennart_Poettering_-_Systemd',
 121                 'upload_date': '20141120',
 122             },
 123             'expected_warnings': [
 124                 'URL could be a direct video link, returning it as such.'
 125             ]
 126         },
 127         # RSS feed
 128         {
 129             'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 130             'info_dict': {
 131                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 132                 'title': 'Zero Punctuation',
 133                 'description': 're:.*groundbreaking video review series.*'
 134             },
 135             'playlist_mincount': 11,
 136         },
 137         # RSS feed with enclosure
 138         {
 139             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
 140             'info_dict': {
 141                 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 142                 'ext': 'm4v',
 143                 'upload_date': '20150228',
 144                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 145             }
 146         },
 147         # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
 148         {
 149             'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
 150             'info_dict': {
 151                 'id': 'smil',
 152                 'ext': 'mp4',
 153                 'title': 'Automatics, robotics and biocybernetics',
 154                 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
 155                 'upload_date': '20130627',
 156                 'formats': 'mincount:16',
 157                 'subtitles': 'mincount:1',
 158             },
 159             'params': {
 160                 'force_generic_extractor': True,
 161                 'skip_download': True,
 162             },
 163         },
 164         # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
 165         {
 166             'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
 167             'info_dict': {
 168                 'id': 'hds',
 169                 'ext': 'flv',
 170                 'title': 'hds',
 171                 'formats': 'mincount:1',
 172             },
 173             'params': {
 174                 'skip_download': True,
 175             },
 176         },
 177         # SMIL from https://www.restudy.dk/video/play/id/1637
 178         {
 179             'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
 180             'info_dict': {
 181                 'id': 'video_1637',
 182                 'ext': 'flv',
 183                 'title': 'video_1637',
 184                 'formats': 'mincount:3',
 185             },
 186             'params': {
 187                 'skip_download': True,
 188             },
 189         },
 190         # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
 191         {
 192             'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
 193             'info_dict': {
 194                 'id': 'smil-service',
 195                 'ext': 'flv',
 196                 'title': 'smil-service',
 197                 'formats': 'mincount:1',
 198             },
 199             'params': {
 200                 'skip_download': True,
 201             },
 202         },
 203         # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
 204         {
 205             'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
 206             'info_dict': {
 207                 'id': '4719370',
 208                 'ext': 'mp4',
 209                 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
 210                 'formats': 'mincount:3',
 211             },
 212             'params': {
 213                 'skip_download': True,
 214             },
 215         },
 216         # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
 217         {
 218             'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
 219             'info_dict': {
 220                 'id': 'mZlp2ctYIUEB',
 221                 'ext': 'mp4',
 222                 'title': 'Tikibad ontruimd wegens brand',
 223                 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
 224                 'thumbnail': 're:^https?://.*\.jpg$',
 225                 'duration': 33,
 226             },
 227             'params': {
 228                 'skip_download': True,
 229             },
 230         },
 231         # MPD from http://dash-mse-test.appspot.com/media.html
 232         {
 233             'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
 234             'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
 235             'info_dict': {
 236                 'id': 'car-20120827-manifest',
 237                 'ext': 'mp4',
 238                 'title': 'car-20120827-manifest',
 239                 'formats': 'mincount:9',
 240             },
 241             'params': {
 242                 'format': 'bestvideo',
 243             },
 244         },
 245         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
 246         {
 247             'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
 248             'info_dict': {
 249                 'id': 'content',
 250                 'ext': 'mp4',
 251                 'title': 'content',
 252                 'formats': 'mincount:8',
 253             },
 254             'params': {
 255                 # m3u8 downloads
 256                 'skip_download': True,
 257             }
 258         },
 259         # m3u8 served with Content-Type: text/plain
 260         {
 261             'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
 262             'info_dict': {
 263                 'id': 'index',
 264                 'ext': 'mp4',
 265                 'title': 'index',
 266                 'upload_date': '20140720',
 267                 'formats': 'mincount:11',
 268             },
 269             'params': {
 270                 # m3u8 downloads
 271                 'skip_download': True,
 272             }
 273         },
 274         # google redirect
 275         {
 276             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
 277             'info_dict': {
 278                 'id': 'cmQHVoWB5FY',
 279                 'ext': 'mp4',
 280                 'upload_date': '20130224',
 281                 'uploader_id': 'TheVerge',
 282                 'description': 're:^Chris Ziegler takes a look at the\.*',
 283                 'uploader': 'The Verge',
 284                 'title': 'First Firefox OS phones side-by-side',
 285             },
 286             'params': {
 287                 'skip_download': False,
 288             }
 289         },
 290         {
 291             # redirect in Refresh HTTP header
 292             'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
 293             'info_dict': {
 294                 'id': 'pO8h3EaFRdo',
 295                 'ext': 'mp4',
 296                 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
 297                 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
 298                 'upload_date': '20150917',
 299                 'uploader_id': 'brtvofficial',
 300                 'uploader': 'Boiler Room',
 301             },
 302             'params': {
 303                 'skip_download': False,
 304             },
 305         },
 306         {
 307             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
 308             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
 309             'info_dict': {
 310                 'id': '13601338388002',
 311                 'ext': 'mp4',
 312                 'uploader': 'www.hodiho.fr',
 313                 'title': 'R\u00e9gis plante sa Jeep',
 314             }
 315         },
 316         # bandcamp page with custom domain
 317         {
 318             'add_ie': ['Bandcamp'],
 319             'url': 'http://bronyrock.com/track/the-pony-mash',
 320             'info_dict': {
 321                 'id': '3235767654',
 322                 'ext': 'mp3',
 323                 'title': 'The Pony Mash',
 324                 'uploader': 'M_Pallante',
 325             },
 326             'skip': 'There is a limit of 200 free downloads / month for the test song',
 327         },
 328         # embedded brightcove video
 329         # it also tests brightcove videos that need to set the 'Referer' in the
 330         # http requests
 331         {
 332             'add_ie': ['BrightcoveLegacy'],
 333             'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
 334             'info_dict': {
 335                 'id': '2765128793001',
 336                 'ext': 'mp4',
 337                 'title': 'Le cours de bourse : l’analyse technique',
 338                 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
 339                 'uploader': 'BFM BUSINESS',
 340             },
 341             'params': {
 342                 'skip_download': True,
 343             },
 344         },
 345         {
 346             # https://github.com/rg3/youtube-dl/issues/2253
 347             'url': 'http://bcove.me/i6nfkrc3',
 348             'md5': '0ba9446db037002366bab3b3eb30c88c',
 349             'info_dict': {
 350                 'id': '3101154703001',
 351                 'ext': 'mp4',
 352                 'title': 'Still no power',
 353                 'uploader': 'thestar.com',
 354                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
 355             },
 356             'add_ie': ['BrightcoveLegacy'],
 357         },
 358         {
 359             'url': 'http://www.championat.com/video/football/v/87/87499.html',
 360             'md5': 'fb973ecf6e4a78a67453647444222983',
 361             'info_dict': {
 362                 'id': '3414141473001',
 363                 'ext': 'mp4',
 364                 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
 365                 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
 366                 'uploader': 'Championat',
 367             },
 368         },
 369         {
 370             # https://github.com/rg3/youtube-dl/issues/3541
 371             'add_ie': ['BrightcoveLegacy'],
 372             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
 373             'info_dict': {
 374                 'id': '3866516442001',
 375                 'ext': 'mp4',
 376                 'title': 'Leer mij vrouwen kennen: Aflevering 1',
 377                 'description': 'Leer mij vrouwen kennen: Aflevering 1',
 378                 'uploader': 'SBS Broadcasting',
 379             },
 380             'skip': 'Restricted to Netherlands',
 381             'params': {
 382                 'skip_download': True,  # m3u8 download
 383             },
 384         },
 385         # ooyala video
 386         {
 387             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
 388             'md5': '166dd577b433b4d4ebfee10b0824d8ff',
 389             'info_dict': {
 390                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
 391                 'ext': 'mp4',
 392                 'title': '2cc213299525360.mov',  # that's what we get
 393                 'duration': 238.231,
 394             },
 395             'add_ie': ['Ooyala'],
 396         },
 397         {
 398             # ooyala video embedded with http://player.ooyala.com/iframe.js
 399             'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
 400             'info_dict': {
 401                 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
 402                 'ext': 'mp4',
 403                 'title': '"Steve Jobs: Man in the Machine" trailer',
 404                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
 405                 'duration': 135.427,
 406             },
 407             'params': {
 408                 'skip_download': True,
 409             },
 410         },
 411         # embed.ly video
 412         {
 413             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
 414             'info_dict': {
 415                 'id': '9ODmcdjQcHQ',
 416                 'ext': 'mp4',
 417                 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
 418                 'upload_date': '20140225',
 419                 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
 420                 'uploader': 'Tested',
 421                 'uploader_id': 'testedcom',
 422             },
 423             # No need to test YoutubeIE here
 424             'params': {
 425                 'skip_download': True,
 426             },
 427         },
 428         # funnyordie embed
 429         {
 430             'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
 431             'info_dict': {
 432                 'id': '18e820ec3f',
 433                 'ext': 'mp4',
 434                 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
 435                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
 436             },
 437         },
 438         # RUTV embed
 439         {
 440             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
 441             'info_dict': {
 442                 'id': '776940',
 443                 'ext': 'mp4',
 444                 'title': 'Охотское море стало целиком российским',
 445                 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
 446             },
 447             'params': {
 448                 # m3u8 download
 449                 'skip_download': True,
 450             },
 451         },
 452         # TVC embed
 453         {
 454             'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
 455             'info_dict': {
 456                 'id': '55304',
 457                 'ext': 'mp4',
 458                 'title': 'Дошкольное воспитание',
 459             },
 460         },
 461         # SportBox embed
 462         {
 463             'url': 'http://www.vestifinance.ru/articles/25753',
 464             'info_dict': {
 465                 'id': '25753',
 466                 'title': 'Вести Экономика ― Прямые трансляции с Форума-выставки "Госзаказ-2013"',
 467             },
 468             'playlist': [{
 469                 'info_dict': {
 470                     'id': '370908',
 471                     'title': 'Госзаказ. День 3',
 472                     'ext': 'mp4',
 473                 }
 474             }, {
 475                 'info_dict': {
 476                     'id': '370905',
 477                     'title': 'Госзаказ. День 2',
 478                     'ext': 'mp4',
 479                 }
 480             }, {
 481                 'info_dict': {
 482                     'id': '370902',
 483                     'title': 'Госзаказ. День 1',
 484                     'ext': 'mp4',
 485                 }
 486             }],
 487             'params': {
 488                 # m3u8 download
 489                 'skip_download': True,
 490             },
 491         },
 492         # Myvi.ru embed
 493         {
 494             'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
 495             'info_dict': {
 496                 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
 497                 'ext': 'mp4',
 498                 'title': 'Ужастики, русский трейлер (2015)',
 499                 'thumbnail': 're:^https?://.*\.jpg$',
 500                 'duration': 153,
 501             }
 502         },
 503         # XHamster embed
 504         {
 505             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
 506             'info_dict': {
 507                 'id': 'showthread',
 508                 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
 509             },
 510             'playlist_mincount': 7,
 511         },
 512         # Embedded TED video
 513         {
 514             'url': 'http://en.support.wordpress.com/videos/ted-talks/',
 515             'md5': '65fdff94098e4a607385a60c5177c638',
 516             'info_dict': {
 517                 'id': '1969',
 518                 'ext': 'mp4',
 519                 'title': 'Hidden miracles of the natural world',
 520                 'uploader': 'Louie Schwartzberg',
 521                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
 522             }
 523         },
 524         # Embedded Ustream video
 525         {
 526             'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
 527             'md5': '27b99cdb639c9b12a79bca876a073417',
 528             'info_dict': {
 529                 'id': '45734260',
 530                 'ext': 'flv',
 531                 'uploader': 'AU SPA:  The NSA and Privacy',
 532                 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
 533             }
 534         },
 535         # nowvideo embed hidden behind percent encoding
 536         {
 537             'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
 538             'md5': '2baf4ddd70f697d94b1c18cf796d5107',
 539             'info_dict': {
 540                 'id': '06e53103ca9aa',
 541                 'ext': 'flv',
 542                 'title': 'Macross Episode 001  Watch Macross Episode 001 onl',
 543                 'description': 'No description',
 544             },
 545         },
 546         # arte embed
 547         {
 548             'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
 549             'md5': '7653032cbb25bf6c80d80f217055fa43',
 550             'info_dict': {
 551                 'id': '048195-004_PLUS7-F',
 552                 'ext': 'flv',
 553                 'title': 'X:enius',
 554                 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
 555                 'upload_date': '20140320',
 556             },
 557             'params': {
 558                 'skip_download': 'Requires rtmpdump'
 559             }
 560         },
 561         # francetv embed
 562         {
 563             'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
 564             'info_dict': {
 565                 'id': 'EV_30231',
 566                 'ext': 'mp4',
 567                 'title': 'Alcaline, le concert avec Calogero',
 568                 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
 569                 'upload_date': '20150226',
 570                 'timestamp': 1424989860,
 571                 'duration': 5400,
 572             },
 573             'params': {
 574                 # m3u8 downloads
 575                 'skip_download': True,
 576             },
 577             'expected_warnings': [
 578                 'Forbidden'
 579             ]
 580         },
 581         # Condé Nast embed
 582         {
 583             'url': 'http://www.wired.com/2014/04/honda-asimo/',
 584             'md5': 'ba0dfe966fa007657bd1443ee672db0f',
 585             'info_dict': {
 586                 'id': '53501be369702d3275860000',
 587                 'ext': 'mp4',
 588                 'title': 'Honda’s  New Asimo Robot Is More Human Than Ever',
 589             }
 590         },
 591         # Dailymotion embed
 592         {
 593             'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
 594             'md5': '441aeeb82eb72c422c7f14ec533999cd',
 595             'info_dict': {
 596                 'id': 'k2mm4bCdJ6CQ2i7c8o2',
 597                 'ext': 'mp4',
 598                 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
 599                 'uploader': 'Spi0n',
 600             },
 601             'add_ie': ['Dailymotion'],
 602         },
 603         # YouTube embed
 604         {
 605             'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
 606             'info_dict': {
 607                 'id': 'FXRb4ykk4S0',
 608                 'ext': 'mp4',
 609                 'title': 'The NBL Auction 2014',
 610                 'uploader': 'BADMINTON England',
 611                 'uploader_id': 'BADMINTONEvents',
 612                 'upload_date': '20140603',
 613                 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
 614             },
 615             'add_ie': ['Youtube'],
 616             'params': {
 617                 'skip_download': True,
 618             }
 619         },
 620         # MTVSercices embed
 621         {
 622             'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
 623             'md5': '35727f82f58c76d996fc188f9755b0d5',
 624             'info_dict': {
 625                 'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
 626                 'ext': 'mp4',
 627                 'title': 'Review',
 628                 'description': 'Mario\'s life in the fast lane has never looked so good.',
 629             },
 630         },
 631         # YouTube embed via <data-embed-url="">
 632         {
 633             'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
 634             'info_dict': {
 635                 'id': '4vAffPZIT44',
 636                 'ext': 'mp4',
 637                 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
 638                 'uploader': 'Gameloft',
 639                 'uploader_id': 'gameloft',
 640                 'upload_date': '20140828',
 641                 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
 642             },
 643             'params': {
 644                 'skip_download': True,
 645             }
 646         },
 647         # Camtasia studio
 648         {
 649             'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
 650             'playlist': [{
 651                 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
 652                 'info_dict': {
 653                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 654                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
 655                     'ext': 'flv',
 656                     'duration': 2235.90,
 657                 }
 658             }, {
 659                 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
 660                 'info_dict': {
 661                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
 662                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
 663                     'ext': 'flv',
 664                     'duration': 2235.93,
 665                 }
 666             }],
 667             'info_dict': {
 668                 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 669             }
 670         },
 671         # Flowplayer
 672         {
 673             'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
 674             'md5': '9d65602bf31c6e20014319c7d07fba27',
 675             'info_dict': {
 676                 'id': '5123ea6d5e5a7',
 677                 'ext': 'mp4',
 678                 'age_limit': 18,
 679                 'uploader': 'www.handjobhub.com',
 680                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
 681             }
 682         },
 683         # Multiple brightcove videos
 684         # https://github.com/rg3/youtube-dl/issues/2283
 685         {
 686             'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
 687             'info_dict': {
 688                 'id': 'always-never',
 689                 'title': 'Always / Never - The New Yorker',
 690             },
 691             'playlist_count': 3,
 692             'params': {
 693                 'extract_flat': False,
 694                 'skip_download': True,
 695             }
 696         },
 697         # MLB embed
 698         {
 699             'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
 700             'md5': '96f09a37e44da40dd083e12d9a683327',
 701             'info_dict': {
 702                 'id': '33322633',
 703                 'ext': 'mp4',
 704                 'title': 'Ump changes call to ball',
 705                 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
 706                 'duration': 48,
 707                 'timestamp': 1401537900,
 708                 'upload_date': '20140531',
 709                 'thumbnail': 're:^https?://.*\.jpg$',
 710             },
 711         },
 712         # Wistia embed
 713         {
 714             'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
 715             'md5': '8788b683c777a5cf25621eaf286d0c23',
 716             'info_dict': {
 717                 'id': '1cfaf6b7ea',
 718                 'ext': 'mov',
 719                 'title': 'md5:51364a8d3d009997ba99656004b5e20d',
 720                 'duration': 643.0,
 721                 'filesize': 182808282,
 722                 'uploader': 'education-portal.com',
 723             },
 724         },
 725         {
 726             'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
 727             'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
 728             'info_dict': {
 729                 'id': 'uxjb0lwrcz',
 730                 'ext': 'mp4',
 731                 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
 732                 'duration': 1715.0,
 733                 'uploader': 'thoughtworks.wistia.com',
 734             },
 735         },
 736         # Soundcloud embed
 737         {
 738             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
 739             'info_dict': {
 740                 'id': '174391317',
 741                 'ext': 'mp3',
 742                 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
 743                 'uploader': 'Sophos Security',
 744                 'title': 'Chet Chat 171 - Oct 29, 2014',
 745                 'upload_date': '20141029',
 746             }
 747         },
 748         # Livestream embed
 749         {
 750             'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
 751             'info_dict': {
 752                 'id': '67864563',
 753                 'ext': 'flv',
 754                 'upload_date': '20141112',
 755                 'title': 'Rosetta #CometLanding webcast HL 10',
 756             }
 757         },
 758         # LazyYT
 759         {
 760             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
 761             'info_dict': {
 762                 'id': '1986',
 763                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
 764             },
 765             'playlist_mincount': 2,
 766         },
 767         # Cinchcast embed
 768         {
 769             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
 770             'info_dict': {
 771                 'id': '7141703',
 772                 'ext': 'mp3',
 773                 'upload_date': '20141126',
 774                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
 775             }
 776         },
 777         # Cinerama player
 778         {
 779             'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
 780             'info_dict': {
 781                 'id': '730m_DandD_1901_512k',
 782                 'ext': 'mp4',
 783                 'uploader': 'www.abc.net.au',
 784                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
 785             }
 786         },
 787         # embedded viddler video
 788         {
 789             'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
 790             'info_dict': {
 791                 'id': '4d03aad9',
 792                 'ext': 'mp4',
 793                 'uploader': 'deadspin',
 794                 'title': 'WALL-TO-GORTAT',
 795                 'timestamp': 1422285291,
 796                 'upload_date': '20150126',
 797             },
 798             'add_ie': ['Viddler'],
 799         },
 800         # Libsyn embed
 801         {
 802             'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
 803             'info_dict': {
 804                 'id': '3377616',
 805                 'ext': 'mp3',
 806                 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
 807                 'description': 'md5:601cb790edd05908957dae8aaa866465',
 808                 'upload_date': '20150220',
 809             },
 810         },
 811         # jwplayer YouTube
 812         {
 813             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
 814             'info_dict': {
 815                 'id': 'Mrj4DVp2zeA',
 816                 'ext': 'mp4',
 817                 'upload_date': '20150212',
 818                 'uploader': 'The National Archives UK',
 819                 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
 820                 'uploader_id': 'NationalArchives08',
 821                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
 822             },
 823         },
 824         # rtl.nl embed
 825         {
 826             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
 827             'playlist_mincount': 5,
 828             'info_dict': {
 829                 'id': 'aanslagen-kopenhagen',
 830                 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
 831             }
 832         },
 833         # Zapiks embed
 834         {
 835             'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
 836             'info_dict': {
 837                 'id': '118046',
 838                 'ext': 'mp4',
 839                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
 840             }
 841         },
 842         # Kaltura embed
 843         {
 844             'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
 845             'info_dict': {
 846                 'id': '1_eergr3h1',
 847                 'ext': 'mp4',
 848                 'upload_date': '20150226',
 849                 'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
 850                 'timestamp': int,
 851                 'title': 'John Carlson Postgame 2/25/15',
 852             },
 853         },
 854         # Kaltura embed (different embed code)
 855         {
 856             'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
 857             'info_dict': {
 858                 'id': '1_a52wc67y',
 859                 'ext': 'flv',
 860                 'upload_date': '20150127',
 861                 'uploader_id': 'PremierMedia',
 862                 'timestamp': int,
 863                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
 864             },
 865         },
 866         # Kaltura embed protected with referrer
 867         {
 868             'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
 869             'info_dict': {
 870                 'id': '1_g4fbemnq',
 871                 'ext': 'mp4',
 872                 'title': 'Violetta - Achter De Schermen - Ruggero',
 873                 'description': 'Achter de schermen met Ruggero',
 874                 'timestamp': 1435133761,
 875                 'upload_date': '20150624',
 876                 'uploader_id': 'echojecka',
 877             },
 878         },
 879         # Eagle.Platform embed (generic URL)
 880         {
 881             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
 882             'info_dict': {
 883                 'id': '227304',
 884                 'ext': 'mp4',
 885                 'title': 'Навальный вышел на свободу',
 886                 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
 887                 'thumbnail': 're:^https?://.*\.jpg$',
 888                 'duration': 87,
 889                 'view_count': int,
 890                 'age_limit': 0,
 891             },
 892         },
 893         # ClipYou (Eagle.Platform) embed (custom URL)
 894         {
 895             'url': 'http://muz-tv.ru/play/7129/',
 896             'info_dict': {
 897                 'id': '12820',
 898                 'ext': 'mp4',
 899                 'title': "'O Sole Mio",
 900                 'thumbnail': 're:^https?://.*\.jpg$',
 901                 'duration': 216,
 902                 'view_count': int,
 903             },
 904         },
 905         # Pladform embed
 906         {
 907             'url': 'http://muz-tv.ru/kinozal/view/7400/',
 908             'info_dict': {
 909                 'id': '100183293',
 910                 'ext': 'mp4',
 911                 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
 912                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
 913                 'thumbnail': 're:^https?://.*\.jpg$',
 914                 'duration': 694,
 915                 'age_limit': 0,
 916             },
 917         },
 918         # Playwire embed
 919         {
 920             'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
 921             'info_dict': {
 922                 'id': '3519514',
 923                 'ext': 'mp4',
 924                 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
 925                 'thumbnail': 're:^https?://.*\.png$',
 926                 'duration': 45.115,
 927             },
 928         },
 929         # 5min embed
 930         {
 931             'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
 932             'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
 933             'info_dict': {
 934                 'id': '518726732',
 935                 'ext': 'mp4',
 936                 'title': 'Facebook Creates "On This Day" | Crunch Report',
 937             },
 938         },
 939         # SVT embed
 940         {
 941             'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
 942             'info_dict': {
 943                 'id': '2900353',
 944                 'ext': 'flv',
 945                 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
 946                 'duration': 27,
 947                 'age_limit': 0,
 948             },
 949         },
 950         # Crooks and Liars embed
 951         {
 952             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
 953             'info_dict': {
 954                 'id': '8RUoRhRi',
 955                 'ext': 'mp4',
 956                 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
 957                 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
 958                 'timestamp': 1428207000,
 959                 'upload_date': '20150405',
 960                 'uploader': 'Heather',
 961             },
 962         },
 963         # Crooks and Liars external embed
 964         {
 965             'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
 966             'info_dict': {
 967                 'id': 'MTE3MjUtMzQ2MzA',
 968                 'ext': 'mp4',
 969                 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
 970                 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
 971                 'timestamp': 1265032391,
 972                 'upload_date': '20100201',
 973                 'uploader': 'Heather',
 974             },
 975         },
 976         # NBC Sports vplayer embed
 977         {
 978             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
 979             'info_dict': {
 980                 'id': 'ln7x1qSThw4k',
 981                 'ext': 'flv',
 982                 'title': "PFT Live: New leader in the 'new-look' defense",
 983                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
 984             },
 985         },
 986         # UDN embed
 987         {
 988             'url': 'http://www.udn.com/news/story/7314/822787',
 989             'md5': 'fd2060e988c326991037b9aff9df21a6',
 990             'info_dict': {
 991                 'id': '300346',
 992                 'ext': 'mp4',
 993                 'title': '中一中男師變性 全校師生力挺',
 994                 'thumbnail': 're:^https?://.*\.jpg$',
 995             }
 996         },
 997         # Ooyala embed
 998         {
 999             'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
1000             'info_dict': {
1001                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
1002                 'ext': 'mp4',
1003                 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
1004                 'title': 'This is what separates the Excel masters from the wannabes',
1005                 'duration': 191.933,
1006             },
1007             'params': {
1008                 # m3u8 downloads
1009                 'skip_download': True,
1010             }
1011         },
1012         # Contains a SMIL manifest
1013         {
1014             'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
1015             'info_dict': {
1016                 'id': 'file',
1017                 'ext': 'flv',
1018                 'title': '+ Football: Lottery Champions League Europe',
1019                 'uploader': 'www.telewebion.com',
1020             },
1021             'params': {
1022                 # rtmpe downloads
1023                 'skip_download': True,
1024             }
1025         },
1026         # Brightcove URL in single quotes
1027         {
1028             'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1029             'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1030             'info_dict': {
1031                 'id': '4255764656001',
1032                 'ext': 'mp4',
1033                 'title': 'SN Presents: Russell Martin, World Citizen',
1034                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1035                 'uploader': 'Rogers Sportsnet',
1036             },
1037         },
1038         # Dailymotion Cloud video
1039         {
1040             'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
1041             'md5': '49444254273501a64675a7e68c502681',
1042             'info_dict': {
1043                 'id': '5585de919473990de4bee11b',
1044                 'ext': 'mp4',
1045                 'title': 'Le débat',
1046                 'thumbnail': 're:^https?://.*\.jpe?g$',
1047             }
1048         },
1049         # OnionStudios embed
1050         {
1051             'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1052             'info_dict': {
1053                 'id': '2855',
1054                 'ext': 'mp4',
1055                 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1056                 'thumbnail': 're:^https?://.*\.jpe?g$',
1057                 'uploader': 'ClickHole',
1058                 'uploader_id': 'clickhole',
1059             }
1060         },
1061         # SnagFilms embed
1062         {
1063             'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1064             'info_dict': {
1065                 'id': '74849a00-85a9-11e1-9660-123139220831',
1066                 'ext': 'mp4',
1067                 'title': '#whilewewatch',
1068             }
1069         },
1070         # AdobeTVVideo embed
1071         {
1072             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1073             'md5': '43662b577c018ad707a63766462b1e87',
1074             'info_dict': {
1075                 'id': '2456',
1076                 'ext': 'mp4',
1077                 'title': 'New experience with Acrobat DC',
1078                 'description': 'New experience with Acrobat DC',
1079                 'duration': 248.667,
1080             },
1081         },
1082         # ScreenwaveMedia embed
1083         {
1084             'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
1085             'md5': '24ace5baba0d35d55c6810b51f34e9e0',
1086             'info_dict': {
1087                 'id': 'cinemasnob-55d26273809dd',
1088                 'ext': 'mp4',
1089                 'title': 'cinemasnob',
1090             },
1091         },
1092         # BrightcoveInPageEmbed embed
1093         {
1094             'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1095             'info_dict': {
1096                 'id': '4238694884001',
1097                 'ext': 'flv',
1098                 'title': 'Tabletop: Dread, Last Thoughts',
1099                 'description': 'Tabletop: Dread, Last Thoughts',
1100                 'duration': 51690,
1101             },
1102         },
1103         # JWPlayer with M3U8
1104         {
1105             'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
1106             'info_dict': {
1107                 'id': 'playlist',
1108                 'ext': 'mp4',
1109                 'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
1110                 'uploader': 'ren.tv',
1111             },
1112             'params': {
1113                 # m3u8 downloads
1114                 'skip_download': True,
1115             }
1116         },
1117         # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
1118         # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
1119         {
1120             'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
1121             'info_dict': {
1122                 'id': '4785848093001',
1123                 'ext': 'mp4',
1124                 'title': 'The Cardinal Pell Interview',
1125                 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
1126                 'uploader': 'GlobeCast Australia - GlobeStream',
1127             },
1128             'params': {
1129                 # m3u8 downloads
1130                 'skip_download': True,
1131             },
1132         },
1133         # Another form of arte.tv embed
1134         {
1135             'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
1136             'md5': '850bfe45417ddf221288c88a0cffe2e2',
1137             'info_dict': {
1138                 'id': '030273-562_PLUS7-F',
1139                 'ext': 'mp4',
1140                 'title': 'ARTE Reportage - Nulle part, en France',
1141                 'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
1142                 'upload_date': '20160409',
1143             },
1144         },
1145         # LiveLeak embed
1146         {
1147             'url': 'http://www.wykop.pl/link/3088787/',
1148             'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
1149             'info_dict': {
1150                 'id': '874_1459135191',
1151                 'ext': 'mp4',
1152                 'title': 'Man shows poor quality of new apartment building',
1153                 'description': 'The wall is like a sand pile.',
1154                 'uploader': 'Lake8737',
1155             }
1156         },
1157     ]
1158
1159     def report_following_redirect(self, new_url):
1160         """Report information extraction."""
1161         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
1162
1163     def _extract_rss(self, url, video_id, doc):
1164         playlist_title = doc.find('./channel/title').text
1165         playlist_desc_el = doc.find('./channel/description')
1166         playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
1167
1168         entries = []
1169         for it in doc.findall('./channel/item'):
1170             next_url = xpath_text(it, 'link', fatal=False)
1171             if not next_url:
1172                 enclosure_nodes = it.findall('./enclosure')
1173                 for e in enclosure_nodes:
1174                     next_url = e.attrib.get('url')
1175                     if next_url:
1176                         break
1177
1178             if not next_url:
1179                 continue
1180
1181             entries.append({
1182                 '_type': 'url',
1183                 'url': next_url,
1184                 'title': it.find('title').text,
1185             })
1186
1187         return {
1188             '_type': 'playlist',
1189             'id': url,
1190             'title': playlist_title,
1191             'description': playlist_desc,
1192             'entries': entries,
1193         }
1194
1195     def _extract_camtasia(self, url, video_id, webpage):
1196         """ Returns None if no camtasia video can be found. """
1197
1198         camtasia_cfg = self._search_regex(
1199             r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
1200             webpage, 'camtasia configuration file', default=None)
1201         if camtasia_cfg is None:
1202             return None
1203
1204         title = self._html_search_meta('DC.title', webpage, fatal=True)
1205
1206         camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
1207         camtasia_cfg = self._download_xml(
1208             camtasia_url, video_id,
1209             note='Downloading camtasia configuration',
1210             errnote='Failed to download camtasia configuration')
1211         fileset_node = camtasia_cfg.find('./playlist/array/fileset')
1212
1213         entries = []
1214         for n in fileset_node.getchildren():
1215             url_n = n.find('./uri')
1216             if url_n is None:
1217                 continue
1218
1219             entries.append({
1220                 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
1221                 'title': '%s - %s' % (title, n.tag),
1222                 'url': compat_urlparse.urljoin(url, url_n.text),
1223                 'duration': float_or_none(n.find('./duration').text),
1224             })
1225
1226         return {
1227             '_type': 'playlist',
1228             'entries': entries,
1229             'title': title,
1230         }
1231
1232     def _real_extract(self, url):
1233         if url.startswith('//'):
1234             return {
1235                 '_type': 'url',
1236                 'url': self.http_scheme() + url,
1237             }
1238
1239         parsed_url = compat_urlparse.urlparse(url)
1240         if not parsed_url.scheme:
1241             default_search = self._downloader.params.get('default_search')
1242             if default_search is None:
1243                 default_search = 'fixup_error'
1244
1245             if default_search in ('auto', 'auto_warning', 'fixup_error'):
1246                 if '/' in url:
1247                     self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
1248                     return self.url_result('http://' + url)
1249                 elif default_search != 'fixup_error':
1250                     if default_search == 'auto_warning':
1251                         if re.match(r'^(?:url|URL)$', url):
1252                             raise ExtractorError(
1253                                 'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
1254                                 expected=True)
1255                         else:
1256                             self._downloader.report_warning(
1257                                 'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
1258                     return self.url_result('ytsearch:' + url)
1259
1260             if default_search in ('error', 'fixup_error'):
1261                 raise ExtractorError(
1262                     '%r is not a valid URL. '
1263                     'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
1264                     % (url, url), expected=True)
1265             else:
1266                 if ':' not in default_search:
1267                     default_search += ':'
1268                 return self.url_result(default_search + url)
1269
1270         url, smuggled_data = unsmuggle_url(url)
1271         force_videoid = None
1272         is_intentional = smuggled_data and smuggled_data.get('to_generic')
1273         if smuggled_data and 'force_videoid' in smuggled_data:
1274             force_videoid = smuggled_data['force_videoid']
1275             video_id = force_videoid
1276         else:
1277             video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
1278
1279         self.to_screen('%s: Requesting header' % video_id)
1280
1281         head_req = HEADRequest(url)
1282         head_response = self._request_webpage(
1283             head_req, video_id,
1284             note=False, errnote='Could not send HEAD request to %s' % url,
1285             fatal=False)
1286
1287         if head_response is not False:
1288             # Check for redirect
1289             new_url = head_response.geturl()
1290             if url != new_url:
1291                 self.report_following_redirect(new_url)
1292                 if force_videoid:
1293                     new_url = smuggle_url(
1294                         new_url, {'force_videoid': force_videoid})
1295                 return self.url_result(new_url)
1296
1297         full_response = None
1298         if head_response is False:
1299             request = sanitized_Request(url)
1300             request.add_header('Accept-Encoding', '*')
1301             full_response = self._request_webpage(request, video_id)
1302             head_response = full_response
1303
1304         info_dict = {
1305             'id': video_id,
1306             'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
1307             'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
1308         }
1309
1310         # Check for direct link to a video
1311         content_type = head_response.headers.get('Content-Type', '').lower()
1312         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
1313         if m:
1314             format_id = m.group('format_id')
1315             if format_id.endswith('mpegurl'):
1316                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
1317             elif format_id == 'f4m':
1318                 formats = self._extract_f4m_formats(url, video_id)
1319             else:
1320                 formats = [{
1321                     'format_id': m.group('format_id'),
1322                     'url': url,
1323                     'vcodec': 'none' if m.group('type') == 'audio' else None
1324                 }]
1325                 info_dict['direct'] = True
1326             self._sort_formats(formats)
1327             info_dict['formats'] = formats
1328             return info_dict
1329
1330         if not self._downloader.params.get('test', False) and not is_intentional:
1331             force = self._downloader.params.get('force_generic_extractor', False)
1332             self._downloader.report_warning(
1333                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
1334
1335         if not full_response:
1336             request = sanitized_Request(url)
1337             # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
1338             # making it impossible to download only chunk of the file (yet we need only 512kB to
1339             # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
1340             # that will always result in downloading the whole file that is not desirable.
1341             # Therefore for extraction pass we have to override Accept-Encoding to any in order
1342             # to accept raw bytes and being able to download only a chunk.
1343             # It may probably better to solve this by checking Content-Type for application/octet-stream
1344             # after HEAD request finishes, but not sure if we can rely on this.
1345             request.add_header('Accept-Encoding', '*')
1346             full_response = self._request_webpage(request, video_id)
1347
1348         first_bytes = full_response.read(512)
1349
1350         # Is it an M3U playlist?
1351         if first_bytes.startswith(b'#EXTM3U'):
1352             info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
1353             self._sort_formats(info_dict['formats'])
1354             return info_dict
1355
1356         # Maybe it's a direct link to a video?
1357         # Be careful not to download the whole thing!
1358         if not is_html(first_bytes):
1359             self._downloader.report_warning(
1360                 'URL could be a direct video link, returning it as such.')
1361             info_dict.update({
1362                 'direct': True,
1363                 'url': url,
1364             })
1365             return info_dict
1366
1367         webpage = self._webpage_read_content(
1368             full_response, url, video_id, prefix=first_bytes)
1369
1370         self.report_extraction(video_id)
1371
1372         # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
1373         try:
1374             doc = compat_etree_fromstring(webpage.encode('utf-8'))
1375             if doc.tag == 'rss':
1376                 return self._extract_rss(url, video_id, doc)
1377             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
1378                 smil = self._parse_smil(doc, url, video_id)
1379                 self._sort_formats(smil['formats'])
1380                 return smil
1381             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
1382                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
1383             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
1384                 info_dict['formats'] = self._parse_mpd_formats(
1385                     doc, video_id, mpd_base_url=url.rpartition('/')[0])
1386                 self._sort_formats(info_dict['formats'])
1387                 return info_dict
1388             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
1389                 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
1390                 self._sort_formats(info_dict['formats'])
1391                 return info_dict
1392         except compat_xml_parse_error:
1393             pass
1394
1395         # Is it a Camtasia project?
1396         camtasia_res = self._extract_camtasia(url, video_id, webpage)
1397         if camtasia_res is not None:
1398             return camtasia_res
1399
1400         # Sometimes embedded video player is hidden behind percent encoding
1401         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1402         # Unescaping the whole page allows to handle those cases in a generic way
1403         webpage = compat_urllib_parse_unquote(webpage)
1404
1405         # it's tempting to parse this further, but you would
1406         # have to take into account all the variations like
1407         #   Video Title - Site Name
1408         #   Site Name | Video Title
1409         #   Video Title - Tagline | Site Name
1410         # and so on and so forth; it's just not practical
1411         video_title = self._html_search_regex(
1412             r'(?s)<title>(.*?)</title>', webpage, 'video title',
1413             default='video')
1414
1415         # Try to detect age limit automatically
1416         age_limit = self._rta_search(webpage)
1417         # And then there are the jokers who advertise that they use RTA,
1418         # but actually don't.
1419         AGE_LIMIT_MARKERS = [
1420             r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1421         ]
1422         if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
1423             age_limit = 18
1424
1425         # video uploader is domain name
1426         video_uploader = self._search_regex(
1427             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
1428
1429         # Helper method
1430         def _playlist_from_matches(matches, getter=None, ie=None):
1431             urlrs = orderedSet(
1432                 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
1433                 for m in matches)
1434             return self.playlist_result(
1435                 urlrs, playlist_id=video_id, playlist_title=video_title)
1436
1437         # Look for Brightcove Legacy Studio embeds
1438         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
1439         if bc_urls:
1440             self.to_screen('Brightcove video detected.')
1441             entries = [{
1442                 '_type': 'url',
1443                 'url': smuggle_url(bc_url, {'Referer': url}),
1444                 'ie_key': 'BrightcoveLegacy'
1445             } for bc_url in bc_urls]
1446
1447             return {
1448                 '_type': 'playlist',
1449                 'title': video_title,
1450                 'id': video_id,
1451                 'entries': entries,
1452             }
1453
1454         # Look for Brightcove New Studio embeds
1455         bc_urls = BrightcoveNewIE._extract_urls(webpage)
1456         if bc_urls:
1457             return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
1458
1459         # Look for embedded rtl.nl player
1460         matches = re.findall(
1461             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
1462             webpage)
1463         if matches:
1464             return _playlist_from_matches(matches, ie='RtlNl')
1465
1466         vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
1467         if vimeo_url is not None:
1468             return self.url_result(vimeo_url)
1469
1470         vid_me_embed_url = self._search_regex(
1471             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
1472             webpage, 'vid.me embed', default=None)
1473         if vid_me_embed_url is not None:
1474             return self.url_result(vid_me_embed_url, 'Vidme')
1475
1476         # Look for embedded YouTube player
1477         matches = re.findall(r'''(?x)
1478             (?:
1479                 <iframe[^>]+?src=|
1480                 data-video-url=|
1481                 <embed[^>]+?src=|
1482                 embedSWF\(?:\s*|
1483                 new\s+SWFObject\(
1484             )
1485             (["\'])
1486                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1487                 (?:embed|v|p)/.+?)
1488             \1''', webpage)
1489         if matches:
1490             return _playlist_from_matches(
1491                 matches, lambda m: unescapeHTML(m[1]))
1492
1493         # Look for lazyYT YouTube embed
1494         matches = re.findall(
1495             r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1496         if matches:
1497             return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1498
1499         # Look for embedded Dailymotion player
1500         matches = re.findall(
1501             r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
1502         if matches:
1503             return _playlist_from_matches(
1504                 matches, lambda m: unescapeHTML(m[1]))
1505
1506         # Look for embedded Dailymotion playlist player (#3822)
1507         m = re.search(
1508             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1509         if m:
1510             playlists = re.findall(
1511                 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1512             if playlists:
1513                 return _playlist_from_matches(
1514                     playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
1515
1516         # Look for embedded Wistia player
1517         match = re.search(
1518             r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1519         if match:
1520             embed_url = self._proto_relative_url(
1521                 unescapeHTML(match.group('url')))
1522             return {
1523                 '_type': 'url_transparent',
1524                 'url': embed_url,
1525                 'ie_key': 'Wistia',
1526                 'uploader': video_uploader,
1527                 'title': video_title,
1528                 'id': video_id,
1529             }
1530
1531         match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1532         if match:
1533             return {
1534                 '_type': 'url_transparent',
1535                 'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
1536                 'ie_key': 'Wistia',
1537                 'uploader': video_uploader,
1538                 'title': video_title,
1539                 'id': match.group('id')
1540             }
1541
1542         # Look for SVT player
1543         svt_url = SVTIE._extract_url(webpage)
1544         if svt_url:
1545             return self.url_result(svt_url, 'SVT')
1546
1547         # Look for embedded condenast player
1548         matches = re.findall(
1549             r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1550             webpage)
1551         if matches:
1552             return {
1553                 '_type': 'playlist',
1554                 'entries': [{
1555                     '_type': 'url',
1556                     'ie_key': 'CondeNast',
1557                     'url': ma,
1558                 } for ma in matches],
1559                 'title': video_title,
1560                 'id': video_id,
1561             }
1562
1563         # Look for Bandcamp pages with custom domain
1564         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1565         if mobj is not None:
1566             burl = unescapeHTML(mobj.group(1))
1567             # Don't set the extractor because it can be a track url or an album
1568             return self.url_result(burl)
1569
1570         # Look for embedded Vevo player
1571         mobj = re.search(
1572             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1573         if mobj is not None:
1574             return self.url_result(mobj.group('url'))
1575
1576         # Look for embedded Viddler player
1577         mobj = re.search(
1578             r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1579             webpage)
1580         if mobj is not None:
1581             return self.url_result(mobj.group('url'))
1582
1583         # Look for NYTimes player
1584         mobj = re.search(
1585             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1586             webpage)
1587         if mobj is not None:
1588             return self.url_result(mobj.group('url'))
1589
1590         # Look for Libsyn player
1591         mobj = re.search(
1592             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1593         if mobj is not None:
1594             return self.url_result(mobj.group('url'))
1595
1596         # Look for Ooyala videos
1597         mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1598                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1599                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1600                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1601         if mobj is not None:
1602             return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
1603
1604         # Look for multiple Ooyala embeds on SBN network websites
1605         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1606         if mobj is not None:
1607             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1608             if embeds:
1609                 return _playlist_from_matches(
1610                     embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
1611
1612         # Look for Aparat videos
1613         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1614         if mobj is not None:
1615             return self.url_result(mobj.group(1), 'Aparat')
1616
1617         # Look for MPORA videos
1618         mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1619         if mobj is not None:
1620             return self.url_result(mobj.group(1), 'Mpora')
1621
1622         # Look for embedded NovaMov-based player
1623         mobj = re.search(
1624             r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
1625                     (?P<url>http://(?:(?:embed|www)\.)?
1626                         (?:novamov\.com|
1627                            nowvideo\.(?:ch|sx|eu|at|ag|co)|
1628                            videoweed\.(?:es|com)|
1629                            movshare\.(?:net|sx|ag)|
1630                            divxstage\.(?:eu|net|ch|co|at|ag))
1631                         /embed\.php.+?)\1''', webpage)
1632         if mobj is not None:
1633             return self.url_result(mobj.group('url'))
1634
1635         # Look for embedded Facebook player
1636         mobj = re.search(
1637             r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
1638         if mobj is not None:
1639             return self.url_result(mobj.group('url'), 'Facebook')
1640
1641         # Look for embedded VK player
1642         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
1643         if mobj is not None:
1644             return self.url_result(mobj.group('url'), 'VK')
1645
1646         # Look for embedded Odnoklassniki player
1647         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
1648         if mobj is not None:
1649             return self.url_result(mobj.group('url'), 'Odnoklassniki')
1650
1651         # Look for embedded ivi player
1652         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
1653         if mobj is not None:
1654             return self.url_result(mobj.group('url'), 'Ivi')
1655
1656         # Look for embedded Huffington Post player
1657         mobj = re.search(
1658             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1659         if mobj is not None:
1660             return self.url_result(mobj.group('url'), 'HuffPost')
1661
1662         # Look for embed.ly
1663         mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
1664         if mobj is not None:
1665             return self.url_result(mobj.group('url'))
1666         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
1667         if mobj is not None:
1668             return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
1669
1670         # Look for funnyordie embed
1671         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
1672         if matches:
1673             return _playlist_from_matches(
1674                 matches, getter=unescapeHTML, ie='FunnyOrDie')
1675
1676         # Look for BBC iPlayer embed
1677         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
1678         if matches:
1679             return _playlist_from_matches(matches, ie='BBCCoUk')
1680
1681         # Look for embedded RUTV player
1682         rutv_url = RUTVIE._extract_url(webpage)
1683         if rutv_url:
1684             return self.url_result(rutv_url, 'RUTV')
1685
1686         # Look for embedded TVC player
1687         tvc_url = TVCIE._extract_url(webpage)
1688         if tvc_url:
1689             return self.url_result(tvc_url, 'TVC')
1690
1691         # Look for embedded SportBox player
1692         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
1693         if sportbox_urls:
1694             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
1695
1696         # Look for embedded PornHub player
1697         pornhub_url = PornHubIE._extract_url(webpage)
1698         if pornhub_url:
1699             return self.url_result(pornhub_url, 'PornHub')
1700
1701         # Look for embedded XHamster player
1702         xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
1703         if xhamster_urls:
1704             return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
1705
1706         # Look for embedded TNAFlixNetwork player
1707         tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
1708         if tnaflix_urls:
1709             return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())
1710
1711         # Look for embedded Tvigle player
1712         mobj = re.search(
1713             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
1714         if mobj is not None:
1715             return self.url_result(mobj.group('url'), 'Tvigle')
1716
1717         # Look for embedded TED player
1718         mobj = re.search(
1719             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
1720         if mobj is not None:
1721             return self.url_result(mobj.group('url'), 'TED')
1722
1723         # Look for embedded Ustream videos
1724         mobj = re.search(
1725             r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
1726         if mobj is not None:
1727             return self.url_result(mobj.group('url'), 'Ustream')
1728
1729         # Look for embedded arte.tv player
1730         mobj = re.search(
1731             r'<(?:script|iframe) [^>]*?src="(?P<url>http://www\.arte\.tv/(?:playerv2/embed|arte_vp/index)[^"]+)"',
1732             webpage)
1733         if mobj is not None:
1734             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
1735
1736         # Look for embedded francetv player
1737         mobj = re.search(
1738             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
1739             webpage)
1740         if mobj is not None:
1741             return self.url_result(mobj.group('url'))
1742
1743         # Look for embedded smotri.com player
1744         smotri_url = SmotriIE._extract_url(webpage)
1745         if smotri_url:
1746             return self.url_result(smotri_url, 'Smotri')
1747
1748         # Look for embedded Myvi.ru player
1749         myvi_url = MyviIE._extract_url(webpage)
1750         if myvi_url:
1751             return self.url_result(myvi_url)
1752
1753         # Look for embedded soundcloud player
1754         mobj = re.search(
1755             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
1756             webpage)
1757         if mobj is not None:
1758             url = unescapeHTML(mobj.group('url'))
1759             return self.url_result(url)
1760
1761         # Look for embedded vulture.com player
1762         mobj = re.search(
1763             r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
1764             webpage)
1765         if mobj is not None:
1766             url = unescapeHTML(mobj.group('url'))
1767             return self.url_result(url, ie='Vulture')
1768
1769         # Look for embedded mtvservices player
1770         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
1771         if mtvservices_url:
1772             return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
1773
1774         # Look for embedded yahoo player
1775         mobj = re.search(
1776             r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
1777             webpage)
1778         if mobj is not None:
1779             return self.url_result(mobj.group('url'), 'Yahoo')
1780
1781         # Look for embedded sbs.com.au player
1782         mobj = re.search(
1783             r'''(?x)
1784             (?:
1785                 <meta\s+property="og:video"\s+content=|
1786                 <iframe[^>]+?src=
1787             )
1788             (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
1789             webpage)
1790         if mobj is not None:
1791             return self.url_result(mobj.group('url'), 'SBS')
1792
1793         # Look for embedded Cinchcast player
1794         mobj = re.search(
1795             r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
1796             webpage)
1797         if mobj is not None:
1798             return self.url_result(mobj.group('url'), 'Cinchcast')
1799
1800         mobj = re.search(
1801             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
1802             webpage)
1803         if not mobj:
1804             mobj = re.search(
1805                 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
1806                 webpage)
1807         if mobj is not None:
1808             return self.url_result(mobj.group('url'), 'MLB')
1809
1810         mobj = re.search(
1811             r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
1812             webpage)
1813         if mobj is not None:
1814             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
1815
1816         mobj = re.search(
1817             r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
1818             webpage)
1819         if mobj is not None:
1820             return self.url_result(mobj.group('url'), 'Livestream')
1821
1822         # Look for Zapiks embed
1823         mobj = re.search(
1824             r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
1825         if mobj is not None:
1826             return self.url_result(mobj.group('url'), 'Zapiks')
1827
1828         # Look for Kaltura embeds
1829         mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
1830                 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
1831         if mobj is not None:
1832             return self.url_result(smuggle_url(
1833                 'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
1834                 {'source_url': url}), 'Kaltura')
1835
1836         # Look for Eagle.Platform embeds
1837         mobj = re.search(
1838             r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
1839         if mobj is not None:
1840             return self.url_result(mobj.group('url'), 'EaglePlatform')
1841
1842         # Look for ClipYou (uses Eagle.Platform) embeds
1843         mobj = re.search(
1844             r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
1845         if mobj is not None:
1846             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
1847
1848         # Look for Pladform embeds
1849         pladform_url = PladformIE._extract_url(webpage)
1850         if pladform_url:
1851             return self.url_result(pladform_url)
1852
1853         # Look for Videomore embeds
1854         videomore_url = VideomoreIE._extract_url(webpage)
1855         if videomore_url:
1856             return self.url_result(videomore_url)
1857
1858         # Look for Playwire embeds
1859         mobj = re.search(
1860             r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
1861         if mobj is not None:
1862             return self.url_result(mobj.group('url'))
1863
1864         # Look for 5min embeds
1865         mobj = re.search(
1866             r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
1867         if mobj is not None:
1868             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
1869
1870         # Look for Crooks and Liars embeds
1871         mobj = re.search(
1872             r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
1873         if mobj is not None:
1874             return self.url_result(mobj.group('url'))
1875
1876         # Look for NBC Sports VPlayer embeds
1877         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
1878         if nbc_sports_url:
1879             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
1880
1881         # Look for Google Drive embeds
1882         google_drive_url = GoogleDriveIE._extract_url(webpage)
1883         if google_drive_url:
1884             return self.url_result(google_drive_url, 'GoogleDrive')
1885
1886         # Look for UDN embeds
1887         mobj = re.search(
1888             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
1889         if mobj is not None:
1890             return self.url_result(
1891                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
1892
1893         # Look for Senate ISVP iframe
1894         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
1895         if senate_isvp_url:
1896             return self.url_result(senate_isvp_url, 'SenateISVP')
1897
1898         # Look for Dailymotion Cloud videos
1899         dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
1900         if dmcloud_url:
1901             return self.url_result(dmcloud_url, 'DailymotionCloud')
1902
1903         # Look for OnionStudios embeds
1904         onionstudios_url = OnionStudiosIE._extract_url(webpage)
1905         if onionstudios_url:
1906             return self.url_result(onionstudios_url)
1907
1908         # Look for SnagFilms embeds
1909         snagfilms_url = SnagFilmsEmbedIE._extract_url(webpage)
1910         if snagfilms_url:
1911             return self.url_result(snagfilms_url)
1912
1913         # Look for JWPlatform embeds
1914         jwplatform_url = JWPlatformIE._extract_url(webpage)
1915         if jwplatform_url:
1916             return self.url_result(jwplatform_url, 'JWPlatform')
1917
1918         # Look for ScreenwaveMedia embeds
1919         mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
1920         if mobj is not None:
1921             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
1922
1923         # Look for Digiteka embeds
1924         digiteka_url = DigitekaIE._extract_url(webpage)
1925         if digiteka_url:
1926             return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
1927
1928         # Look for Limelight embeds
1929         mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
1930         if mobj:
1931             lm = {
1932                 'Media': 'media',
1933                 'Channel': 'channel',
1934                 'ChannelList': 'channel_list',
1935             }
1936             return self.url_result('limelight:%s:%s' % (
1937                 lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
1938
1939         # Look for AdobeTVVideo embeds
1940         mobj = re.search(
1941             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
1942             webpage)
1943         if mobj is not None:
1944             return self.url_result(
1945                 self._proto_relative_url(unescapeHTML(mobj.group(1))),
1946                 'AdobeTVVideo')
1947
1948         # Look for Vine embeds
1949         mobj = re.search(
1950             r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
1951             webpage)
1952         if mobj is not None:
1953             return self.url_result(
1954                 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
1955
1956         # Look for Instagram embeds
1957         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
1958         if instagram_embed_url is not None:
1959             return self.url_result(instagram_embed_url, InstagramIE.ie_key())
1960
1961         # Look for LiveLeak embeds
1962         liveleak_url = LiveLeakIE._extract_url(webpage)
1963         if liveleak_url:
1964             return self.url_result(liveleak_url, 'LiveLeak')
1965
1966         def check_video(vurl):
1967             if YoutubeIE.suitable(vurl):
1968                 return True
1969             vpath = compat_urlparse.urlparse(vurl).path
1970             vext = determine_ext(vpath)
1971             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
1972
1973         def filter_video(urls):
1974             return list(filter(check_video, urls))
1975
1976         # Start with something easy: JW Player in SWFObject
1977         found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
1978         if not found:
1979             # Look for gorilla-vid style embedding
1980             found = filter_video(re.findall(r'''(?sx)
1981                 (?:
1982                     jw_plugins|
1983                     JWPlayerOptions|
1984                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
1985                 )
1986                 .*?
1987                 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
1988         if not found:
1989             # Broaden the search a little bit
1990             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
1991         if not found:
1992             # Broaden the findall a little bit: JWPlayer JS loader
1993             found = filter_video(re.findall(
1994                 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
1995         if not found:
1996             # Flow player
1997             found = filter_video(re.findall(r'''(?xs)
1998                 flowplayer\("[^"]+",\s*
1999                     \{[^}]+?\}\s*,
2000                     \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
2001                         ["']?url["']?\s*:\s*["']([^"']+)["']
2002             ''', webpage))
2003         if not found:
2004             # Cinerama player
2005             found = re.findall(
2006                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
2007         if not found:
2008             # Try to find twitter cards info
2009             found = filter_video(re.findall(
2010                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
2011         if not found:
2012             # We look for Open Graph info:
2013             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
2014             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
2015             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
2016             if m_video_type is not None:
2017                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
2018         if not found:
2019             # HTML5 video
2020             found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
2021         if not found:
2022             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
2023             found = re.search(
2024                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
2025                 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
2026                 webpage)
2027             if not found:
2028                 # Look also in Refresh HTTP header
2029                 refresh_header = head_response.headers.get('Refresh')
2030                 if refresh_header:
2031                     # In python 2 response HTTP headers are bytestrings
2032                     if sys.version_info < (3, 0) and isinstance(refresh_header, str):
2033                         refresh_header = refresh_header.decode('iso-8859-1')
2034                     found = re.search(REDIRECT_REGEX, refresh_header)
2035             if found:
2036                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
2037                 self.report_following_redirect(new_url)
2038                 return {
2039                     '_type': 'url',
2040                     'url': new_url,
2041                 }
2042         if not found:
2043             raise UnsupportedError(url)
2044
2045         entries = []
2046         for video_url in found:
2047             video_url = video_url.replace('\\/', '/')
2048             video_url = compat_urlparse.urljoin(url, video_url)
2049             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
2050
2051             # Sometimes, jwplayer extraction will result in a YouTube URL
2052             if YoutubeIE.suitable(video_url):
2053                 entries.append(self.url_result(video_url, 'Youtube'))
2054                 continue
2055
2056             # here's a fun little line of code for you:
2057             video_id = os.path.splitext(video_id)[0]
2058
2059             entry_info_dict = {
2060                 'id': video_id,
2061                 'uploader': video_uploader,
2062                 'title': video_title,
2063                 'age_limit': age_limit,
2064             }
2065
2066             ext = determine_ext(video_url)
2067             if ext == 'smil':
2068                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
2069             elif ext == 'xspf':
2070                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
2071             elif ext == 'm3u8':
2072                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
2073             elif ext == 'mpd':
2074                 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
2075             elif ext == 'f4m':
2076                 entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
2077             else:
2078                 entry_info_dict['url'] = video_url
2079
2080             if entry_info_dict.get('formats'):
2081                 self._sort_formats(entry_info_dict['formats'])
2082
2083             entries.append(entry_info_dict)
2084
2085         if len(entries) == 1:
2086             return entries[0]
2087         else:
2088             for num, e in enumerate(entries, start=1):
2089                 # 'url' results don't have a title
2090                 if e.get('title') is not None:
2091                     e['title'] = '%s (%d)' % (e['title'], num)
2092             return {
2093                 '_type': 'playlist',
2094                 'entries': entries,
2095             }