_ Git - youtube-dl/blob - youtube_dl/extractor/generic.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import os
   6 import re
   7 import sys
   8
   9 from .common import InfoExtractor
  10 from .youtube import YoutubeIE
  11 from ..compat import (
  12     compat_etree_fromstring,
  13     compat_str,
  14     compat_urllib_parse_unquote,
  15     compat_urlparse,
  16     compat_xml_parse_error,
  17 )
  18 from ..utils import (
  19     determine_ext,
  20     ExtractorError,
  21     float_or_none,
  22     HEADRequest,
  23     is_html,
  24     js_to_json,
  25     KNOWN_EXTENSIONS,
  26     mimetype2ext,
  27     orderedSet,
  28     sanitized_Request,
  29     smuggle_url,
  30     unescapeHTML,
  31     unified_strdate,
  32     unsmuggle_url,
  33     UnsupportedError,
  34     xpath_text,
  35 )
  36 from .commonprotocols import RtmpIE
  37 from .brightcove import (
  38     BrightcoveLegacyIE,
  39     BrightcoveNewIE,
  40 )
  41 from .nexx import (
  42     NexxIE,
  43     NexxEmbedIE,
  44 )
  45 from .nbc import NBCSportsVPlayerIE
  46 from .ooyala import OoyalaIE
  47 from .rutv import RUTVIE
  48 from .tvc import TVCIE
  49 from .sportbox import SportBoxEmbedIE
  50 from .smotri import SmotriIE
  51 from .myvi import MyviIE
  52 from .condenast import CondeNastIE
  53 from .udn import UDNEmbedIE
  54 from .senateisvp import SenateISVPIE
  55 from .svt import SVTIE
  56 from .pornhub import PornHubIE
  57 from .xhamster import XHamsterEmbedIE
  58 from .tnaflix import TNAFlixNetworkEmbedIE
  59 from .drtuber import DrTuberIE
  60 from .redtube import RedTubeIE
  61 from .vimeo import VimeoIE
  62 from .dailymotion import DailymotionIE
  63 from .dailymail import DailyMailIE
  64 from .onionstudios import OnionStudiosIE
  65 from .viewlift import ViewLiftEmbedIE
  66 from .mtv import MTVServicesEmbeddedIE
  67 from .pladform import PladformIE
  68 from .videomore import VideomoreIE
  69 from .webcaster import WebcasterFeedIE
  70 from .googledrive import GoogleDriveIE
  71 from .jwplatform import JWPlatformIE
  72 from .digiteka import DigitekaIE
  73 from .arkena import ArkenaIE
  74 from .instagram import InstagramIE
  75 from .liveleak import LiveLeakIE
  76 from .threeqsdn import ThreeQSDNIE
  77 from .theplatform import ThePlatformIE
  78 from .vessel import VesselIE
  79 from .kaltura import KalturaIE
  80 from .eagleplatform import EaglePlatformIE
  81 from .facebook import FacebookIE
  82 from .soundcloud import SoundcloudIE
  83 from .tunein import TuneInBaseIE
  84 from .vbox7 import Vbox7IE
  85 from .dbtv import DBTVIE
  86 from .piksel import PikselIE
  87 from .videa import VideaIE
  88 from .twentymin import TwentyMinutenIE
  89 from .ustream import UstreamIE
  90 from .openload import OpenloadIE
  91 from .videopress import VideoPressIE
  92 from .rutube import RutubeIE
  93 from .limelight import LimelightBaseIE
  94 from .anvato import AnvatoIE
  95 from .washingtonpost import WashingtonPostIE
  96 from .wistia import WistiaIE
  97 from .mediaset import MediasetIE
  98 from .joj import JojIE
  99 from .megaphone import MegaphoneIE
 100 from .vzaar import VzaarIE
 101 from .channel9 import Channel9IE
 102 from .vshare import VShareIE
 103 from .mediasite import MediasiteIE
 104 from .springboardplatform import SpringboardPlatformIE
 105 from .yapfiles import YapFilesIE
 106
 107
 108 class GenericIE(InfoExtractor):
 109     IE_DESC = 'Generic downloader that works on some sites'
 110     _VALID_URL = r'.*'
 111     IE_NAME = 'generic'
 112     _TESTS = [
 113         # Direct link to a video
 114         {
 115             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
 116             'md5': '67d406c2bcb6af27fa886f31aa934bbe',
 117             'info_dict': {
 118                 'id': 'trailer',
 119                 'ext': 'mp4',
 120                 'title': 'trailer',
 121                 'upload_date': '20100513',
 122             }
 123         },
 124         # Direct link to media delivered compressed (until Accept-Encoding is *)
 125         {
 126             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
 127             'md5': '128c42e68b13950268b648275386fc74',
 128             'info_dict': {
 129                 'id': 'FictionJunction-Parallel_Hearts',
 130                 'ext': 'flac',
 131                 'title': 'FictionJunction-Parallel_Hearts',
 132                 'upload_date': '20140522',
 133             },
 134             'expected_warnings': [
 135                 'URL could be a direct video link, returning it as such.'
 136             ],
 137             'skip': 'URL invalid',
 138         },
 139         # Direct download with broken HEAD
 140         {
 141             'url': 'http://ai-radio.org:8000/radio.opus',
 142             'info_dict': {
 143                 'id': 'radio',
 144                 'ext': 'opus',
 145                 'title': 'radio',
 146             },
 147             'params': {
 148                 'skip_download': True,  # infinite live stream
 149             },
 150             'expected_warnings': [
 151                 r'501.*Not Implemented',
 152                 r'400.*Bad Request',
 153             ],
 154         },
 155         # Direct link with incorrect MIME type
 156         {
 157             'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 158             'md5': '4ccbebe5f36706d85221f204d7eb5913',
 159             'info_dict': {
 160                 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
 161                 'id': '5_Lennart_Poettering_-_Systemd',
 162                 'ext': 'webm',
 163                 'title': '5_Lennart_Poettering_-_Systemd',
 164                 'upload_date': '20141120',
 165             },
 166             'expected_warnings': [
 167                 'URL could be a direct video link, returning it as such.'
 168             ]
 169         },
 170         # RSS feed
 171         {
 172             'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 173             'info_dict': {
 174                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
 175                 'title': 'Zero Punctuation',
 176                 'description': 're:.*groundbreaking video review series.*'
 177             },
 178             'playlist_mincount': 11,
 179         },
 180         # RSS feed with enclosure
 181         {
 182             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
 183             'info_dict': {
 184                 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 185                 'ext': 'm4v',
 186                 'upload_date': '20150228',
 187                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
 188             }
 189         },
 190         # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
 191         {
 192             'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
 193             'info_dict': {
 194                 'id': 'smil',
 195                 'ext': 'mp4',
 196                 'title': 'Automatics, robotics and biocybernetics',
 197                 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
 198                 'upload_date': '20130627',
 199                 'formats': 'mincount:16',
 200                 'subtitles': 'mincount:1',
 201             },
 202             'params': {
 203                 'force_generic_extractor': True,
 204                 'skip_download': True,
 205             },
 206         },
 207         # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
 208         {
 209             'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
 210             'info_dict': {
 211                 'id': 'hds',
 212                 'ext': 'flv',
 213                 'title': 'hds',
 214                 'formats': 'mincount:1',
 215             },
 216             'params': {
 217                 'skip_download': True,
 218             },
 219         },
 220         # SMIL from https://www.restudy.dk/video/play/id/1637
 221         {
 222             'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
 223             'info_dict': {
 224                 'id': 'video_1637',
 225                 'ext': 'flv',
 226                 'title': 'video_1637',
 227                 'formats': 'mincount:3',
 228             },
 229             'params': {
 230                 'skip_download': True,
 231             },
 232         },
 233         # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
 234         {
 235             'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
 236             'info_dict': {
 237                 'id': 'smil-service',
 238                 'ext': 'flv',
 239                 'title': 'smil-service',
 240                 'formats': 'mincount:1',
 241             },
 242             'params': {
 243                 'skip_download': True,
 244             },
 245         },
 246         # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
 247         {
 248             'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
 249             'info_dict': {
 250                 'id': '4719370',
 251                 'ext': 'mp4',
 252                 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
 253                 'formats': 'mincount:3',
 254             },
 255             'params': {
 256                 'skip_download': True,
 257             },
 258         },
 259         # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
 260         {
 261             'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
 262             'info_dict': {
 263                 'id': 'mZlp2ctYIUEB',
 264                 'ext': 'mp4',
 265                 'title': 'Tikibad ontruimd wegens brand',
 266                 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
 267                 'thumbnail': r're:^https?://.*\.jpg$',
 268                 'duration': 33,
 269             },
 270             'params': {
 271                 'skip_download': True,
 272             },
 273         },
 274         # MPD from http://dash-mse-test.appspot.com/media.html
 275         {
 276             'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
 277             'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
 278             'info_dict': {
 279                 'id': 'car-20120827-manifest',
 280                 'ext': 'mp4',
 281                 'title': 'car-20120827-manifest',
 282                 'formats': 'mincount:9',
 283                 'upload_date': '20130904',
 284             },
 285             'params': {
 286                 'format': 'bestvideo',
 287             },
 288         },
 289         # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
 290         {
 291             'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
 292             'info_dict': {
 293                 'id': 'content',
 294                 'ext': 'mp4',
 295                 'title': 'content',
 296                 'formats': 'mincount:8',
 297             },
 298             'params': {
 299                 # m3u8 downloads
 300                 'skip_download': True,
 301             },
 302             'skip': 'video gone',
 303         },
 304         # m3u8 served with Content-Type: text/plain
 305         {
 306             'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
 307             'info_dict': {
 308                 'id': 'index',
 309                 'ext': 'mp4',
 310                 'title': 'index',
 311                 'upload_date': '20140720',
 312                 'formats': 'mincount:11',
 313             },
 314             'params': {
 315                 # m3u8 downloads
 316                 'skip_download': True,
 317             },
 318             'skip': 'video gone',
 319         },
 320         # google redirect
 321         {
 322             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
 323             'info_dict': {
 324                 'id': 'cmQHVoWB5FY',
 325                 'ext': 'mp4',
 326                 'upload_date': '20130224',
 327                 'uploader_id': 'TheVerge',
 328                 'description': r're:^Chris Ziegler takes a look at the\.*',
 329                 'uploader': 'The Verge',
 330                 'title': 'First Firefox OS phones side-by-side',
 331             },
 332             'params': {
 333                 'skip_download': False,
 334             }
 335         },
 336         {
 337             # redirect in Refresh HTTP header
 338             'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
 339             'info_dict': {
 340                 'id': 'pO8h3EaFRdo',
 341                 'ext': 'mp4',
 342                 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
 343                 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
 344                 'upload_date': '20150917',
 345                 'uploader_id': 'brtvofficial',
 346                 'uploader': 'Boiler Room',
 347             },
 348             'params': {
 349                 'skip_download': False,
 350             },
 351         },
 352         {
 353             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
 354             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
 355             'info_dict': {
 356                 'id': '13601338388002',
 357                 'ext': 'mp4',
 358                 'uploader': 'www.hodiho.fr',
 359                 'title': 'R\u00e9gis plante sa Jeep',
 360             }
 361         },
 362         # bandcamp page with custom domain
 363         {
 364             'add_ie': ['Bandcamp'],
 365             'url': 'http://bronyrock.com/track/the-pony-mash',
 366             'info_dict': {
 367                 'id': '3235767654',
 368                 'ext': 'mp3',
 369                 'title': 'The Pony Mash',
 370                 'uploader': 'M_Pallante',
 371             },
 372             'skip': 'There is a limit of 200 free downloads / month for the test song',
 373         },
 374         {
 375             # embedded brightcove video
 376             # it also tests brightcove videos that need to set the 'Referer'
 377             # in the http requests
 378             'add_ie': ['BrightcoveLegacy'],
 379             'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
 380             'info_dict': {
 381                 'id': '2765128793001',
 382                 'ext': 'mp4',
 383                 'title': 'Le cours de bourse : l’analyse technique',
 384                 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
 385                 'uploader': 'BFM BUSINESS',
 386             },
 387             'params': {
 388                 'skip_download': True,
 389             },
 390         },
 391         {
 392             # embedded with itemprop embedURL and video id spelled as `idVideo`
 393             'add_id': ['BrightcoveLegacy'],
 394             'url': 'http://bfmbusiness.bfmtv.com/mediaplayer/chroniques/olivier-delamarche/',
 395             'info_dict': {
 396                 'id': '5255628253001',
 397                 'ext': 'mp4',
 398                 'title': 'md5:37c519b1128915607601e75a87995fc0',
 399                 'description': 'md5:37f7f888b434bb8f8cc8dbd4f7a4cf26',
 400                 'uploader': 'BFM BUSINESS',
 401                 'uploader_id': '876450612001',
 402                 'timestamp': 1482255315,
 403                 'upload_date': '20161220',
 404             },
 405             'params': {
 406                 'skip_download': True,
 407             },
 408         },
 409         {
 410             # https://github.com/rg3/youtube-dl/issues/2253
 411             'url': 'http://bcove.me/i6nfkrc3',
 412             'md5': '0ba9446db037002366bab3b3eb30c88c',
 413             'info_dict': {
 414                 'id': '3101154703001',
 415                 'ext': 'mp4',
 416                 'title': 'Still no power',
 417                 'uploader': 'thestar.com',
 418                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
 419             },
 420             'add_ie': ['BrightcoveLegacy'],
 421             'skip': 'video gone',
 422         },
 423         {
 424             'url': 'http://www.championat.com/video/football/v/87/87499.html',
 425             'md5': 'fb973ecf6e4a78a67453647444222983',
 426             'info_dict': {
 427                 'id': '3414141473001',
 428                 'ext': 'mp4',
 429                 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
 430                 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
 431                 'uploader': 'Championat',
 432             },
 433         },
 434         {
 435             # https://github.com/rg3/youtube-dl/issues/3541
 436             'add_ie': ['BrightcoveLegacy'],
 437             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
 438             'info_dict': {
 439                 'id': '3866516442001',
 440                 'ext': 'mp4',
 441                 'title': 'Leer mij vrouwen kennen: Aflevering 1',
 442                 'description': 'Leer mij vrouwen kennen: Aflevering 1',
 443                 'uploader': 'SBS Broadcasting',
 444             },
 445             'skip': 'Restricted to Netherlands',
 446             'params': {
 447                 'skip_download': True,  # m3u8 download
 448             },
 449         },
 450         {
 451             # Brightcove video in <iframe>
 452             'url': 'http://www.un.org/chinese/News/story.asp?NewsID=27724',
 453             'md5': '36d74ef5e37c8b4a2ce92880d208b968',
 454             'info_dict': {
 455                 'id': '5360463607001',
 456                 'ext': 'mp4',
 457                 'title': '叙利亚失明儿童在废墟上演唱《心跳》  呼吁获得正常童年生活',
 458                 'description': '联合国儿童基金会中东和北非区域大使、作曲家扎德·迪拉尼（Zade Dirani）在3月15日叙利亚冲突爆发7周年纪念日之际发布了为叙利亚谱写的歌曲《心跳》（HEARTBEAT），为受到六年冲突影响的叙利亚儿童发出强烈呐喊，呼吁世界做出共同努力，使叙利亚儿童重新获得享有正常童年生活的权利。',
 459                 'uploader': 'United Nations',
 460                 'uploader_id': '1362235914001',
 461                 'timestamp': 1489593889,
 462                 'upload_date': '20170315',
 463             },
 464             'add_ie': ['BrightcoveLegacy'],
 465         },
 466         {
 467             # Brightcove with alternative playerID key
 468             'url': 'http://www.nature.com/nmeth/journal/v9/n7/fig_tab/nmeth.2062_SV1.html',
 469             'info_dict': {
 470                 'id': 'nmeth.2062_SV1',
 471                 'title': 'Simultaneous multiview imaging of the Drosophila syncytial blastoderm : Quantitative high-speed imaging of entire developing embryos with simultaneous multiview light-sheet microscopy : Nature Methods : Nature Research',
 472             },
 473             'playlist': [{
 474                 'info_dict': {
 475                     'id': '2228375078001',
 476                     'ext': 'mp4',
 477                     'title': 'nmeth.2062-sv1',
 478                     'description': 'nmeth.2062-sv1',
 479                     'timestamp': 1363357591,
 480                     'upload_date': '20130315',
 481                     'uploader': 'Nature Publishing Group',
 482                     'uploader_id': '1964492299001',
 483                 },
 484             }],
 485         },
 486         {
 487             # Brightcove with UUID in videoPlayer
 488             'url': 'http://www8.hp.com/cn/zh/home.html',
 489             'info_dict': {
 490                 'id': '5255815316001',
 491                 'ext': 'mp4',
 492                 'title': 'Sprocket Video - China',
 493                 'description': 'Sprocket Video - China',
 494                 'uploader': 'HP-Video Gallery',
 495                 'timestamp': 1482263210,
 496                 'upload_date': '20161220',
 497                 'uploader_id': '1107601872001',
 498             },
 499             'params': {
 500                 'skip_download': True,  # m3u8 download
 501             },
 502             'skip': 'video rotates...weekly?',
 503         },
 504         {
 505             # Brightcove:new type [2].
 506             'url': 'http://www.delawaresportszone.com/video-st-thomas-more-earns-first-trip-to-basketball-semis',
 507             'md5': '2b35148fcf48da41c9fb4591650784f3',
 508             'info_dict': {
 509                 'id': '5348741021001',
 510                 'ext': 'mp4',
 511                 'upload_date': '20170306',
 512                 'uploader_id': '4191638492001',
 513                 'timestamp': 1488769918,
 514                 'title': 'VIDEO:  St. Thomas More earns first trip to basketball semis',
 515
 516             },
 517         },
 518         {
 519             # Alternative brightcove <video> attributes
 520             'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',
 521             'info_dict': {
 522                 'id': '81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche',
 523                 'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche, Extraits : toutes les vidéos avec Télé-Loisirs",
 524             },
 525             'playlist': [{
 526                 'md5': '732d22ba3d33f2f3fc253c39f8f36523',
 527                 'info_dict': {
 528                     'id': '5311302538001',
 529                     'ext': 'mp4',
 530                     'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche",
 531                     'description': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche (France 2, 5 février 2017)",
 532                     'timestamp': 1486321708,
 533                     'upload_date': '20170205',
 534                     'uploader_id': '800000640001',
 535                 },
 536                 'only_matching': True,
 537             }],
 538         },
 539         {
 540             # Brightcove with UUID in videoPlayer
 541             'url': 'http://www8.hp.com/cn/zh/home.html',
 542             'info_dict': {
 543                 'id': '5255815316001',
 544                 'ext': 'mp4',
 545                 'title': 'Sprocket Video - China',
 546                 'description': 'Sprocket Video - China',
 547                 'uploader': 'HP-Video Gallery',
 548                 'timestamp': 1482263210,
 549                 'upload_date': '20161220',
 550                 'uploader_id': '1107601872001',
 551             },
 552             'params': {
 553                 'skip_download': True,  # m3u8 download
 554             },
 555         },
 556         # ooyala video
 557         {
 558             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
 559             'md5': '166dd577b433b4d4ebfee10b0824d8ff',
 560             'info_dict': {
 561                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
 562                 'ext': 'mp4',
 563                 'title': '2cc213299525360.mov',  # that's what we get
 564                 'duration': 238.231,
 565             },
 566             'add_ie': ['Ooyala'],
 567         },
 568         {
 569             # ooyala video embedded with http://player.ooyala.com/iframe.js
 570             'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
 571             'info_dict': {
 572                 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
 573                 'ext': 'mp4',
 574                 'title': '"Steve Jobs: Man in the Machine" trailer',
 575                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
 576                 'duration': 135.427,
 577             },
 578             'params': {
 579                 'skip_download': True,
 580             },
 581             'skip': 'movie expired',
 582         },
 583         # ooyala video embedded with http://player.ooyala.com/static/v4/production/latest/core.min.js
 584         {
 585             'url': 'http://wnep.com/2017/07/22/steampunk-fest-comes-to-honesdale/',
 586             'info_dict': {
 587                 'id': 'lwYWYxYzE6V5uJMjNGyKtwwiw9ZJD7t2',
 588                 'ext': 'mp4',
 589                 'title': 'Steampunk Fest Comes to Honesdale',
 590                 'duration': 43.276,
 591             },
 592             'params': {
 593                 'skip_download': True,
 594             }
 595         },
 596         # embed.ly video
 597         {
 598             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
 599             'info_dict': {
 600                 'id': '9ODmcdjQcHQ',
 601                 'ext': 'mp4',
 602                 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
 603                 'upload_date': '20140225',
 604                 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
 605                 'uploader': 'Tested',
 606                 'uploader_id': 'testedcom',
 607             },
 608             # No need to test YoutubeIE here
 609             'params': {
 610                 'skip_download': True,
 611             },
 612         },
 613         # funnyordie embed
 614         {
 615             'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
 616             'info_dict': {
 617                 'id': '18e820ec3f',
 618                 'ext': 'mp4',
 619                 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
 620                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
 621             },
 622             # HEAD requests lead to endless 301, while GET is OK
 623             'expected_warnings': ['301'],
 624         },
 625         # RUTV embed
 626         {
 627             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
 628             'info_dict': {
 629                 'id': '776940',
 630                 'ext': 'mp4',
 631                 'title': 'Охотское море стало целиком российским',
 632                 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
 633             },
 634             'params': {
 635                 # m3u8 download
 636                 'skip_download': True,
 637             },
 638         },
 639         # TVC embed
 640         {
 641             'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
 642             'info_dict': {
 643                 'id': '55304',
 644                 'ext': 'mp4',
 645                 'title': 'Дошкольное воспитание',
 646             },
 647         },
 648         # SportBox embed
 649         {
 650             'url': 'http://www.vestifinance.ru/articles/25753',
 651             'info_dict': {
 652                 'id': '25753',
 653                 'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
 654             },
 655             'playlist': [{
 656                 'info_dict': {
 657                     'id': '370908',
 658                     'title': 'Госзаказ. День 3',
 659                     'ext': 'mp4',
 660                 }
 661             }, {
 662                 'info_dict': {
 663                     'id': '370905',
 664                     'title': 'Госзаказ. День 2',
 665                     'ext': 'mp4',
 666                 }
 667             }, {
 668                 'info_dict': {
 669                     'id': '370902',
 670                     'title': 'Госзаказ. День 1',
 671                     'ext': 'mp4',
 672                 }
 673             }],
 674             'params': {
 675                 # m3u8 download
 676                 'skip_download': True,
 677             },
 678         },
 679         # Myvi.ru embed
 680         {
 681             'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
 682             'info_dict': {
 683                 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
 684                 'ext': 'mp4',
 685                 'title': 'Ужастики, русский трейлер (2015)',
 686                 'thumbnail': r're:^https?://.*\.jpg$',
 687                 'duration': 153,
 688             }
 689         },
 690         # XHamster embed
 691         {
 692             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
 693             'info_dict': {
 694                 'id': 'showthread',
 695                 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
 696             },
 697             'playlist_mincount': 7,
 698             # This forum does not allow <iframe> syntaxes anymore
 699             # Now HTML tags are displayed as-is
 700             'skip': 'No videos on this page',
 701         },
 702         # Embedded TED video
 703         {
 704             'url': 'http://en.support.wordpress.com/videos/ted-talks/',
 705             'md5': '65fdff94098e4a607385a60c5177c638',
 706             'info_dict': {
 707                 'id': '1969',
 708                 'ext': 'mp4',
 709                 'title': 'Hidden miracles of the natural world',
 710                 'uploader': 'Louie Schwartzberg',
 711                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
 712             }
 713         },
 714         # nowvideo embed hidden behind percent encoding
 715         {
 716             'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
 717             'md5': '2baf4ddd70f697d94b1c18cf796d5107',
 718             'info_dict': {
 719                 'id': '06e53103ca9aa',
 720                 'ext': 'flv',
 721                 'title': 'Macross Episode 001  Watch Macross Episode 001 onl',
 722                 'description': 'No description',
 723             },
 724         },
 725         # arte embed
 726         {
 727             'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
 728             'md5': '7653032cbb25bf6c80d80f217055fa43',
 729             'info_dict': {
 730                 'id': '048195-004_PLUS7-F',
 731                 'ext': 'flv',
 732                 'title': 'X:enius',
 733                 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
 734                 'upload_date': '20140320',
 735             },
 736             'params': {
 737                 'skip_download': 'Requires rtmpdump'
 738             },
 739             'skip': 'video gone',
 740         },
 741         # francetv embed
 742         {
 743             'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
 744             'info_dict': {
 745                 'id': 'EV_30231',
 746                 'ext': 'mp4',
 747                 'title': 'Alcaline, le concert avec Calogero',
 748                 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
 749                 'upload_date': '20150226',
 750                 'timestamp': 1424989860,
 751                 'duration': 5400,
 752             },
 753             'params': {
 754                 # m3u8 downloads
 755                 'skip_download': True,
 756             },
 757             'expected_warnings': [
 758                 'Forbidden'
 759             ]
 760         },
 761         # Condé Nast embed
 762         {
 763             'url': 'http://www.wired.com/2014/04/honda-asimo/',
 764             'md5': 'ba0dfe966fa007657bd1443ee672db0f',
 765             'info_dict': {
 766                 'id': '53501be369702d3275860000',
 767                 'ext': 'mp4',
 768                 'title': 'Honda’s  New Asimo Robot Is More Human Than Ever',
 769             }
 770         },
 771         # Dailymotion embed
 772         {
 773             'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
 774             'md5': '441aeeb82eb72c422c7f14ec533999cd',
 775             'info_dict': {
 776                 'id': 'k2mm4bCdJ6CQ2i7c8o2',
 777                 'ext': 'mp4',
 778                 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
 779                 'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
 780                 'uploader': 'Spi0n',
 781                 'uploader_id': 'xgditw',
 782                 'upload_date': '20140425',
 783                 'timestamp': 1398441542,
 784             },
 785             'add_ie': ['Dailymotion'],
 786         },
 787         # DailyMail embed
 788         {
 789             'url': 'http://www.bumm.sk/krimi/2017/07/05/biztonsagi-kamera-buktatta-le-az-agg-ferfit-utlegelo-apolot',
 790             'info_dict': {
 791                 'id': '1495629',
 792                 'ext': 'mp4',
 793                 'title': 'Care worker punches elderly dementia patient in head 11 times',
 794                 'description': 'md5:3a743dee84e57e48ec68bf67113199a5',
 795             },
 796             'add_ie': ['DailyMail'],
 797             'params': {
 798                 'skip_download': True,
 799             },
 800         },
 801         # YouTube embed
 802         {
 803             'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
 804             'info_dict': {
 805                 'id': 'FXRb4ykk4S0',
 806                 'ext': 'mp4',
 807                 'title': 'The NBL Auction 2014',
 808                 'uploader': 'BADMINTON England',
 809                 'uploader_id': 'BADMINTONEvents',
 810                 'upload_date': '20140603',
 811                 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
 812             },
 813             'add_ie': ['Youtube'],
 814             'params': {
 815                 'skip_download': True,
 816             }
 817         },
 818         # MTVSercices embed
 819         {
 820             'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
 821             'md5': 'ca1aef97695ef2c1d6973256a57e5252',
 822             'info_dict': {
 823                 'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
 824                 'ext': 'mp4',
 825                 'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
 826                 'description': 'Two valets share their love for movie star Liam Neesons.',
 827                 'timestamp': 1349922600,
 828                 'upload_date': '20121011',
 829             },
 830         },
 831         # YouTube embed via <data-embed-url="">
 832         {
 833             'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
 834             'info_dict': {
 835                 'id': '4vAffPZIT44',
 836                 'ext': 'mp4',
 837                 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
 838                 'uploader': 'Gameloft',
 839                 'uploader_id': 'gameloft',
 840                 'upload_date': '20140828',
 841                 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
 842             },
 843             'params': {
 844                 'skip_download': True,
 845             }
 846         },
 847         # YouTube <object> embed
 848         {
 849             'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
 850             'md5': '516718101ec834f74318df76259fb3cc',
 851             'info_dict': {
 852                 'id': 'msN87y-iEx0',
 853                 'ext': 'webm',
 854                 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
 855                 'upload_date': '20080526',
 856                 'description': 'md5:0ffc78ea3f01b2e2c247d5f8d1d3c18d',
 857                 'uploader': 'Christopher Sykes',
 858                 'uploader_id': 'ChristopherJSykes',
 859             },
 860             'add_ie': ['Youtube'],
 861         },
 862         # Camtasia studio
 863         {
 864             'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
 865             'playlist': [{
 866                 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
 867                 'info_dict': {
 868                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 869                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
 870                     'ext': 'flv',
 871                     'duration': 2235.90,
 872                 }
 873             }, {
 874                 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
 875                 'info_dict': {
 876                     'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
 877                     'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
 878                     'ext': 'flv',
 879                     'duration': 2235.93,
 880                 }
 881             }],
 882             'info_dict': {
 883                 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
 884             }
 885         },
 886         # Flowplayer
 887         {
 888             'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
 889             'md5': '9d65602bf31c6e20014319c7d07fba27',
 890             'info_dict': {
 891                 'id': '5123ea6d5e5a7',
 892                 'ext': 'mp4',
 893                 'age_limit': 18,
 894                 'uploader': 'www.handjobhub.com',
 895                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
 896             }
 897         },
 898         # Multiple brightcove videos
 899         # https://github.com/rg3/youtube-dl/issues/2283
 900         {
 901             'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
 902             'info_dict': {
 903                 'id': 'always-never',
 904                 'title': 'Always / Never - The New Yorker',
 905             },
 906             'playlist_count': 3,
 907             'params': {
 908                 'extract_flat': False,
 909                 'skip_download': True,
 910             }
 911         },
 912         # MLB embed
 913         {
 914             'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
 915             'md5': '96f09a37e44da40dd083e12d9a683327',
 916             'info_dict': {
 917                 'id': '33322633',
 918                 'ext': 'mp4',
 919                 'title': 'Ump changes call to ball',
 920                 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
 921                 'duration': 48,
 922                 'timestamp': 1401537900,
 923                 'upload_date': '20140531',
 924                 'thumbnail': r're:^https?://.*\.jpg$',
 925             },
 926         },
 927         # Wistia embed
 928         {
 929             'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
 930             'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
 931             'info_dict': {
 932                 'id': '6e2wtrbdaf',
 933                 'ext': 'mov',
 934                 'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
 935                 'description': 'a Paywall Videos video from Remilon',
 936                 'duration': 644.072,
 937                 'uploader': 'study.com',
 938                 'timestamp': 1459678540,
 939                 'upload_date': '20160403',
 940                 'filesize': 24687186,
 941             },
 942         },
 943         {
 944             'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
 945             'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
 946             'info_dict': {
 947                 'id': 'uxjb0lwrcz',
 948                 'ext': 'mp4',
 949                 'title': 'Conversation about Hexagonal Rails Part 1',
 950                 'description': 'a Martin Fowler video from ThoughtWorks',
 951                 'duration': 1715.0,
 952                 'uploader': 'thoughtworks.wistia.com',
 953                 'timestamp': 1401832161,
 954                 'upload_date': '20140603',
 955             },
 956         },
 957         # Wistia standard embed (async)
 958         {
 959             'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
 960             'info_dict': {
 961                 'id': '807fafadvk',
 962                 'ext': 'mp4',
 963                 'title': 'Drip Brennan Dunn Workshop',
 964                 'description': 'a JV Webinars video from getdrip-1',
 965                 'duration': 4986.95,
 966                 'timestamp': 1463607249,
 967                 'upload_date': '20160518',
 968             },
 969             'params': {
 970                 'skip_download': True,
 971             }
 972         },
 973         # Soundcloud embed
 974         {
 975             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
 976             'info_dict': {
 977                 'id': '174391317',
 978                 'ext': 'mp3',
 979                 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
 980                 'uploader': 'Sophos Security',
 981                 'title': 'Chet Chat 171 - Oct 29, 2014',
 982                 'upload_date': '20141029',
 983             }
 984         },
 985         # Soundcloud multiple embeds
 986         {
 987             'url': 'http://www.guitarplayer.com/lessons/1014/legato-workout-one-hour-to-more-fluid-performance---tab/52809',
 988             'info_dict': {
 989                 'id': '52809',
 990                 'title': 'Guitar Essentials: Legato Workout—One-Hour to Fluid Performance  | TAB + AUDIO',
 991             },
 992             'playlist_mincount': 7,
 993         },
 994         # TuneIn station embed
 995         {
 996             'url': 'http://radiocnrv.com/promouvoir-radio-cnrv/',
 997             'info_dict': {
 998                 'id': '204146',
 999                 'ext': 'mp3',
1000                 'title': 'CNRV',
1001                 'location': 'Paris, France',
1002                 'is_live': True,
1003             },
1004             'params': {
1005                 # Live stream
1006                 'skip_download': True,
1007             },
1008         },
1009         # Livestream embed
1010         {
1011             'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
1012             'info_dict': {
1013                 'id': '67864563',
1014                 'ext': 'flv',
1015                 'upload_date': '20141112',
1016                 'title': 'Rosetta #CometLanding webcast HL 10',
1017             }
1018         },
1019         # Another Livestream embed, without 'new.' in URL
1020         {
1021             'url': 'https://www.freespeech.org/',
1022             'info_dict': {
1023                 'id': '123537347',
1024                 'ext': 'mp4',
1025                 'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
1026             },
1027             'params': {
1028                 # Live stream
1029                 'skip_download': True,
1030             },
1031         },
1032         # LazyYT
1033         {
1034             'url': 'https://skiplagged.com/',
1035             'info_dict': {
1036                 'id': 'skiplagged',
1037                 'title': 'Skiplagged: The smart way to find cheap flights',
1038             },
1039             'playlist_mincount': 1,
1040             'add_ie': ['Youtube'],
1041         },
1042         # Cinchcast embed
1043         {
1044             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
1045             'info_dict': {
1046                 'id': '7141703',
1047                 'ext': 'mp3',
1048                 'upload_date': '20141126',
1049                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
1050             }
1051         },
1052         # Cinerama player
1053         {
1054             'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
1055             'info_dict': {
1056                 'id': '730m_DandD_1901_512k',
1057                 'ext': 'mp4',
1058                 'uploader': 'www.abc.net.au',
1059                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
1060             }
1061         },
1062         # embedded viddler video
1063         {
1064             'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
1065             'info_dict': {
1066                 'id': '4d03aad9',
1067                 'ext': 'mp4',
1068                 'uploader': 'deadspin',
1069                 'title': 'WALL-TO-GORTAT',
1070                 'timestamp': 1422285291,
1071                 'upload_date': '20150126',
1072             },
1073             'add_ie': ['Viddler'],
1074         },
1075         # Libsyn embed
1076         {
1077             'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
1078             'info_dict': {
1079                 'id': '3377616',
1080                 'ext': 'mp3',
1081                 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
1082                 'description': 'md5:601cb790edd05908957dae8aaa866465',
1083                 'upload_date': '20150220',
1084             },
1085             'skip': 'All The Daily Show URLs now redirect to http://www.cc.com/shows/',
1086         },
1087         # jwplayer YouTube
1088         {
1089             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
1090             'info_dict': {
1091                 'id': 'Mrj4DVp2zeA',
1092                 'ext': 'mp4',
1093                 'upload_date': '20150212',
1094                 'uploader': 'The National Archives UK',
1095                 'description': 'md5:8078af856dca76edc42910b61273dbbf',
1096                 'uploader_id': 'NationalArchives08',
1097                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
1098             },
1099         },
1100         # jwplayer rtmp
1101         {
1102             'url': 'http://www.suffolk.edu/sjc/live.php',
1103             'info_dict': {
1104                 'id': 'live',
1105                 'ext': 'flv',
1106                 'title': 'Massachusetts Supreme Judicial Court Oral Arguments',
1107                 'uploader': 'www.suffolk.edu',
1108             },
1109             'params': {
1110                 'skip_download': True,
1111             },
1112             'skip': 'Only has video a few mornings per month, see http://www.suffolk.edu/sjc/',
1113         },
1114         # Complex jwplayer
1115         {
1116             'url': 'http://www.indiedb.com/games/king-machine/videos',
1117             'info_dict': {
1118                 'id': 'videos',
1119                 'ext': 'mp4',
1120                 'title': 'king machine trailer 1',
1121                 'description': 'Browse King Machine videos & audio for sweet media. Your eyes will thank you.',
1122                 'thumbnail': r're:^https?://.*\.jpg$',
1123             },
1124         },
1125         {
1126             # JWPlayer config passed as variable
1127             'url': 'http://www.txxx.com/videos/3326530/ariele/',
1128             'info_dict': {
1129                 'id': '3326530_hq',
1130                 'ext': 'mp4',
1131                 'title': 'ARIELE | Tube Cup',
1132                 'uploader': 'www.txxx.com',
1133                 'age_limit': 18,
1134             },
1135             'params': {
1136                 'skip_download': True,
1137             }
1138         },
1139         {
1140             # JWPlatform iframe
1141             'url': 'https://www.mediaite.com/tv/dem-senator-claims-gary-cohn-faked-a-bad-connection-during-trump-call-to-get-him-off-the-phone/',
1142             'md5': 'ca00a040364b5b439230e7ebfd02c4e9',
1143             'info_dict': {
1144                 'id': 'O0c5JcKT',
1145                 'ext': 'mp4',
1146                 'upload_date': '20171122',
1147                 'timestamp': 1511366290,
1148                 'title': 'Dem Senator Claims Gary Cohn Faked a Bad Connection During Trump Call to Get Him Off the Phone',
1149             },
1150             'add_ie': [JWPlatformIE.ie_key()],
1151         },
1152         {
1153             # Video.js embed, multiple formats
1154             'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
1155             'info_dict': {
1156                 'id': 'yygqldloqIk',
1157                 'ext': 'mp4',
1158                 'title': 'SolidWorks. Урок 6 Настройка чертежа',
1159                 'description': 'md5:baf95267792646afdbf030e4d06b2ab3',
1160                 'upload_date': '20130314',
1161                 'uploader': 'PROстое3D',
1162                 'uploader_id': 'PROstoe3D',
1163             },
1164             'params': {
1165                 'skip_download': True,
1166             },
1167         },
1168         {
1169             # Video.js embed, single format
1170             'url': 'https://www.vooplayer.com/v3/watch/watch.php?v=NzgwNTg=',
1171             'info_dict': {
1172                 'id': 'watch',
1173                 'ext': 'mp4',
1174                 'title': 'Step 1 -  Good Foundation',
1175                 'description': 'md5:d1e7ff33a29fc3eb1673d6c270d344f4',
1176             },
1177             'params': {
1178                 'skip_download': True,
1179             },
1180         },
1181         # rtl.nl embed
1182         {
1183             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
1184             'playlist_mincount': 5,
1185             'info_dict': {
1186                 'id': 'aanslagen-kopenhagen',
1187                 'title': 'Aanslagen Kopenhagen',
1188             }
1189         },
1190         # Zapiks embed
1191         {
1192             'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
1193             'info_dict': {
1194                 'id': '118046',
1195                 'ext': 'mp4',
1196                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
1197             }
1198         },
1199         # Kaltura embed (different embed code)
1200         {
1201             'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
1202             'info_dict': {
1203                 'id': '1_a52wc67y',
1204                 'ext': 'flv',
1205                 'upload_date': '20150127',
1206                 'uploader_id': 'PremierMedia',
1207                 'timestamp': int,
1208                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
1209             },
1210         },
1211         # Kaltura embed with single quotes
1212         {
1213             'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
1214             'info_dict': {
1215                 'id': '0_izeg5utt',
1216                 'ext': 'mp4',
1217                 'title': '35871',
1218                 'timestamp': 1355743100,
1219                 'upload_date': '20121217',
1220                 'uploader_id': 'batchUser',
1221             },
1222             'add_ie': ['Kaltura'],
1223         },
1224         {
1225             # Kaltura embedded via quoted entry_id
1226             'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
1227             'info_dict': {
1228                 'id': '0_utuok90b',
1229                 'ext': 'mp4',
1230                 'title': '06_matthew_brender_raj_dutt',
1231                 'timestamp': 1466638791,
1232                 'upload_date': '20160622',
1233             },
1234             'add_ie': ['Kaltura'],
1235             'expected_warnings': [
1236                 'Could not send HEAD request'
1237             ],
1238             'params': {
1239                 'skip_download': True,
1240             }
1241         },
1242         {
1243             # Kaltura embedded, some fileExt broken (#11480)
1244             'url': 'http://www.cornell.edu/video/nima-arkani-hamed-standard-models-of-particle-physics',
1245             'info_dict': {
1246                 'id': '1_sgtvehim',
1247                 'ext': 'mp4',
1248                 'title': 'Our "Standard Models" of particle physics and cosmology',
1249                 'description': 'md5:67ea74807b8c4fea92a6f38d6d323861',
1250                 'timestamp': 1321158993,
1251                 'upload_date': '20111113',
1252                 'uploader_id': 'kps1',
1253             },
1254             'add_ie': ['Kaltura'],
1255         },
1256         {
1257             # Kaltura iframe embed
1258             'url': 'http://www.gsd.harvard.edu/event/i-m-pei-a-centennial-celebration/',
1259             'md5': 'ae5ace8eb09dc1a35d03b579a9c2cc44',
1260             'info_dict': {
1261                 'id': '0_f2cfbpwy',
1262                 'ext': 'mp4',
1263                 'title': 'I. M. Pei: A Centennial Celebration',
1264                 'description': 'md5:1db8f40c69edc46ca180ba30c567f37c',
1265                 'upload_date': '20170403',
1266                 'uploader_id': 'batchUser',
1267                 'timestamp': 1491232186,
1268             },
1269             'add_ie': ['Kaltura'],
1270         },
1271         # EaglePlatform embed (generic URL)
1272         {
1273             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
1274             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
1275             'info_dict': {
1276                 'id': '227304',
1277                 'ext': 'mp4',
1278                 'title': 'Навальный вышел на свободу',
1279                 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
1280                 'thumbnail': r're:^https?://.*\.jpg$',
1281                 'duration': 87,
1282                 'view_count': int,
1283                 'age_limit': 0,
1284             },
1285             'params': {
1286                 'skip_download': True,
1287             },
1288         },
1289         # referrer protected EaglePlatform embed
1290         {
1291             'url': 'https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/',
1292             'info_dict': {
1293                 'id': '582306',
1294                 'ext': 'mp4',
1295                 'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
1296                 'thumbnail': r're:^https?://.*\.jpg$',
1297                 'duration': 3382,
1298                 'view_count': int,
1299             },
1300             'params': {
1301                 'skip_download': True,
1302             },
1303         },
1304         # ClipYou (EaglePlatform) embed (custom URL)
1305         {
1306             'url': 'http://muz-tv.ru/play/7129/',
1307             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
1308             'info_dict': {
1309                 'id': '12820',
1310                 'ext': 'mp4',
1311                 'title': "'O Sole Mio",
1312                 'thumbnail': r're:^https?://.*\.jpg$',
1313                 'duration': 216,
1314                 'view_count': int,
1315             },
1316             'params': {
1317                 'skip_download': True,
1318             },
1319             'skip': 'This video is unavailable.',
1320         },
1321         # Pladform embed
1322         {
1323             'url': 'http://muz-tv.ru/kinozal/view/7400/',
1324             'info_dict': {
1325                 'id': '100183293',
1326                 'ext': 'mp4',
1327                 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
1328                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
1329                 'thumbnail': r're:^https?://.*\.jpg$',
1330                 'duration': 694,
1331                 'age_limit': 0,
1332             },
1333             'skip': 'HTTP Error 404: Not Found',
1334         },
1335         # Playwire embed
1336         {
1337             'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
1338             'info_dict': {
1339                 'id': '3519514',
1340                 'ext': 'mp4',
1341                 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
1342                 'thumbnail': r're:^https?://.*\.png$',
1343                 'duration': 45.115,
1344             },
1345         },
1346         # 5min embed
1347         {
1348             'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
1349             'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
1350             'info_dict': {
1351                 'id': '518726732',
1352                 'ext': 'mp4',
1353                 'title': 'Facebook Creates "On This Day" | Crunch Report',
1354                 'description': 'Amazon updates Fire TV line, Tesla\'s Model X spotted in the wild',
1355                 'timestamp': 1427237531,
1356                 'uploader': 'Crunch Report',
1357                 'upload_date': '20150324',
1358             },
1359             'params': {
1360                 # m3u8 download
1361                 'skip_download': True,
1362             },
1363         },
1364         # SVT embed
1365         {
1366             'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
1367             'info_dict': {
1368                 'id': '2900353',
1369                 'ext': 'flv',
1370                 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
1371                 'duration': 27,
1372                 'age_limit': 0,
1373             },
1374         },
1375         # Crooks and Liars embed
1376         {
1377             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
1378             'info_dict': {
1379                 'id': '8RUoRhRi',
1380                 'ext': 'mp4',
1381                 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
1382                 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
1383                 'timestamp': 1428207000,
1384                 'upload_date': '20150405',
1385                 'uploader': 'Heather',
1386             },
1387         },
1388         # Crooks and Liars external embed
1389         {
1390             'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
1391             'info_dict': {
1392                 'id': 'MTE3MjUtMzQ2MzA',
1393                 'ext': 'mp4',
1394                 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
1395                 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
1396                 'timestamp': 1265032391,
1397                 'upload_date': '20100201',
1398                 'uploader': 'Heather',
1399             },
1400         },
1401         # NBC Sports vplayer embed
1402         {
1403             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
1404             'info_dict': {
1405                 'id': 'ln7x1qSThw4k',
1406                 'ext': 'flv',
1407                 'title': "PFT Live: New leader in the 'new-look' defense",
1408                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
1409                 'uploader': 'NBCU-SPORTS',
1410                 'upload_date': '20140107',
1411                 'timestamp': 1389118457,
1412             },
1413             'skip': 'Invalid Page URL',
1414         },
1415         # NBC News embed
1416         {
1417             'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
1418             'md5': '1aa589c675898ae6d37a17913cf68d66',
1419             'info_dict': {
1420                 'id': 'x_dtl_oa_LettermanliftPR_160608',
1421                 'ext': 'mp4',
1422                 'title': 'David Letterman: A Preview',
1423                 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
1424                 'upload_date': '20160609',
1425                 'timestamp': 1465431544,
1426                 'uploader': 'NBCU-NEWS',
1427             },
1428         },
1429         # UDN embed
1430         {
1431             'url': 'https://video.udn.com/news/300346',
1432             'md5': 'fd2060e988c326991037b9aff9df21a6',
1433             'info_dict': {
1434                 'id': '300346',
1435                 'ext': 'mp4',
1436                 'title': '中一中男師變性 全校師生力挺',
1437                 'thumbnail': r're:^https?://.*\.jpg$',
1438             },
1439             'params': {
1440                 # m3u8 download
1441                 'skip_download': True,
1442             },
1443             'expected_warnings': ['Failed to parse JSON Expecting value'],
1444         },
1445         # Ooyala embed
1446         {
1447             'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
1448             'info_dict': {
1449                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
1450                 'ext': 'mp4',
1451                 'description': 'Index/Match versus VLOOKUP.',
1452                 'title': 'This is what separates the Excel masters from the wannabes',
1453                 'duration': 191.933,
1454             },
1455             'params': {
1456                 # m3u8 downloads
1457                 'skip_download': True,
1458             }
1459         },
1460         # Brightcove URL in single quotes
1461         {
1462             'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1463             'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1464             'info_dict': {
1465                 'id': '4255764656001',
1466                 'ext': 'mp4',
1467                 'title': 'SN Presents: Russell Martin, World Citizen',
1468                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1469                 'uploader': 'Rogers Sportsnet',
1470                 'uploader_id': '1704050871',
1471                 'upload_date': '20150525',
1472                 'timestamp': 1432570283,
1473             },
1474         },
1475         # OnionStudios embed
1476         {
1477             'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1478             'info_dict': {
1479                 'id': '2855',
1480                 'ext': 'mp4',
1481                 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1482                 'thumbnail': r're:^https?://.*\.jpe?g$',
1483                 'uploader': 'ClickHole',
1484                 'uploader_id': 'clickhole',
1485             }
1486         },
1487         # SnagFilms embed
1488         {
1489             'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1490             'info_dict': {
1491                 'id': '74849a00-85a9-11e1-9660-123139220831',
1492                 'ext': 'mp4',
1493                 'title': '#whilewewatch',
1494             }
1495         },
1496         # AdobeTVVideo embed
1497         {
1498             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1499             'md5': '43662b577c018ad707a63766462b1e87',
1500             'info_dict': {
1501                 'id': '2456',
1502                 'ext': 'mp4',
1503                 'title': 'New experience with Acrobat DC',
1504                 'description': 'New experience with Acrobat DC',
1505                 'duration': 248.667,
1506             },
1507         },
1508         # BrightcoveInPageEmbed embed
1509         {
1510             'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1511             'info_dict': {
1512                 'id': '4238694884001',
1513                 'ext': 'flv',
1514                 'title': 'Tabletop: Dread, Last Thoughts',
1515                 'description': 'Tabletop: Dread, Last Thoughts',
1516                 'duration': 51690,
1517             },
1518         },
1519         # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
1520         # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
1521         {
1522             'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
1523             'info_dict': {
1524                 'id': '4785848093001',
1525                 'ext': 'mp4',
1526                 'title': 'The Cardinal Pell Interview',
1527                 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
1528                 'uploader': 'GlobeCast Australia - GlobeStream',
1529                 'uploader_id': '2733773828001',
1530                 'upload_date': '20160304',
1531                 'timestamp': 1457083087,
1532             },
1533             'params': {
1534                 # m3u8 downloads
1535                 'skip_download': True,
1536             },
1537         },
1538         {
1539             # Brightcove embed with whitespace around attribute names
1540             'url': 'http://www.stack.com/video/3167554373001/learn-to-hit-open-three-pointers-with-damian-lillard-s-baseline-drift-drill',
1541             'info_dict': {
1542                 'id': '3167554373001',
1543                 'ext': 'mp4',
1544                 'title': "Learn to Hit Open Three-Pointers With Damian Lillard's Baseline Drift Drill",
1545                 'description': 'md5:57bacb0e0f29349de4972bfda3191713',
1546                 'uploader_id': '1079349493',
1547                 'upload_date': '20140207',
1548                 'timestamp': 1391810548,
1549             },
1550             'params': {
1551                 'skip_download': True,
1552             },
1553         },
1554         # Another form of arte.tv embed
1555         {
1556             'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
1557             'md5': '850bfe45417ddf221288c88a0cffe2e2',
1558             'info_dict': {
1559                 'id': '030273-562_PLUS7-F',
1560                 'ext': 'mp4',
1561                 'title': 'ARTE Reportage - Nulle part, en France',
1562                 'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
1563                 'upload_date': '20160409',
1564             },
1565         },
1566         # LiveLeak embed
1567         {
1568             'url': 'http://www.wykop.pl/link/3088787/',
1569             'md5': '7619da8c820e835bef21a1efa2a0fc71',
1570             'info_dict': {
1571                 'id': '874_1459135191',
1572                 'ext': 'mp4',
1573                 'title': 'Man shows poor quality of new apartment building',
1574                 'description': 'The wall is like a sand pile.',
1575                 'uploader': 'Lake8737',
1576             },
1577             'add_ie': [LiveLeakIE.ie_key()],
1578         },
1579         # Another LiveLeak embed pattern (#13336)
1580         {
1581             'url': 'https://milo.yiannopoulos.net/2017/06/concealed-carry-robbery/',
1582             'info_dict': {
1583                 'id': '2eb_1496309988',
1584                 'ext': 'mp4',
1585                 'title': 'Thief robs place where everyone was armed',
1586                 'description': 'md5:694d73ee79e535953cf2488562288eee',
1587                 'uploader': 'brazilwtf',
1588             },
1589             'add_ie': [LiveLeakIE.ie_key()],
1590         },
1591         # Duplicated embedded video URLs
1592         {
1593             'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
1594             'info_dict': {
1595                 'id': '149298443_480_16c25b74_2',
1596                 'ext': 'mp4',
1597                 'title': 'vs. Blue Orange Spring Game',
1598                 'uploader': 'www.hudl.com',
1599             },
1600         },
1601         # twitter:player:stream embed
1602         {
1603             'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
1604             'info_dict': {
1605                 'id': 'master',
1606                 'ext': 'mp4',
1607                 'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
1608                 'uploader': 'www.rtl.be',
1609             },
1610             'params': {
1611                 # m3u8 downloads
1612                 'skip_download': True,
1613             },
1614         },
1615         # twitter:player embed
1616         {
1617             'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
1618             'md5': 'a3e0df96369831de324f0778e126653c',
1619             'info_dict': {
1620                 'id': '4909620399001',
1621                 'ext': 'mp4',
1622                 'title': 'What Do Black Holes Sound Like?',
1623                 'description': 'what do black holes sound like',
1624                 'upload_date': '20160524',
1625                 'uploader_id': '29913724001',
1626                 'timestamp': 1464107587,
1627                 'uploader': 'TheAtlantic',
1628             },
1629             'add_ie': ['BrightcoveLegacy'],
1630         },
1631         # Facebook <iframe> embed
1632         {
1633             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
1634             'md5': 'fbcde74f534176ecb015849146dd3aee',
1635             'info_dict': {
1636                 'id': '599637780109885',
1637                 'ext': 'mp4',
1638                 'title': 'Facebook video #599637780109885',
1639             },
1640         },
1641         # Facebook <iframe> embed, plugin video
1642         {
1643             'url': 'http://5pillarsuk.com/2017/06/07/tariq-ramadan-disagrees-with-pr-exercise-by-imams-refusing-funeral-prayers-for-london-attackers/',
1644             'info_dict': {
1645                 'id': '1754168231264132',
1646                 'ext': 'mp4',
1647                 'title': 'About the Imams and Religious leaders refusing to perform funeral prayers for...',
1648                 'uploader': 'Tariq Ramadan (official)',
1649                 'timestamp': 1496758379,
1650                 'upload_date': '20170606',
1651             },
1652             'params': {
1653                 'skip_download': True,
1654             },
1655         },
1656         # Facebook API embed
1657         {
1658             'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
1659             'md5': 'a47372ee61b39a7b90287094d447d94e',
1660             'info_dict': {
1661                 'id': '10153467542406923',
1662                 'ext': 'mp4',
1663                 'title': 'Facebook video #10153467542406923',
1664             },
1665         },
1666         # Wordpress "YouTube Video Importer" plugin
1667         {
1668             'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
1669             'md5': 'd16797741b560b485194eddda8121b48',
1670             'info_dict': {
1671                 'id': 'HNTXWDXV9Is',
1672                 'ext': 'mp4',
1673                 'title': 'Blue Devils Drumline Stanford lot 2016',
1674                 'upload_date': '20160627',
1675                 'uploader_id': 'GENOCIDE8GENERAL10',
1676                 'uploader': 'cylus cyrus',
1677             },
1678         },
1679         {
1680             # video stored on custom kaltura server
1681             'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
1682             'md5': '537617d06e64dfed891fa1593c4b30cc',
1683             'info_dict': {
1684                 'id': '0_1iotm5bh',
1685                 'ext': 'mp4',
1686                 'title': 'Elecciones británicas: 5 lecciones para Rajoy',
1687                 'description': 'md5:435a89d68b9760b92ce67ed227055f16',
1688                 'uploader_id': 'videos.expansion@el-mundo.net',
1689                 'upload_date': '20150429',
1690                 'timestamp': 1430303472,
1691             },
1692             'add_ie': ['Kaltura'],
1693         },
1694         {
1695             # Non-standard Vimeo embed
1696             'url': 'https://openclassrooms.com/courses/understanding-the-web',
1697             'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
1698             'info_dict': {
1699                 'id': '148867247',
1700                 'ext': 'mp4',
1701                 'title': 'Understanding the web - Teaser',
1702                 'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
1703                 'upload_date': '20151214',
1704                 'uploader': 'OpenClassrooms',
1705                 'uploader_id': 'openclassrooms',
1706             },
1707             'add_ie': ['Vimeo'],
1708         },
1709         {
1710             # generic vimeo embed that requires original URL passed as Referer
1711             'url': 'http://racing4everyone.eu/2016/07/30/formula-1-2016-round12-germany/',
1712             'only_matching': True,
1713         },
1714         {
1715             'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
1716             'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
1717             'info_dict': {
1718                 'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
1719                 'ext': 'mp4',
1720                 'title': 'Big Buck Bunny',
1721                 'description': 'Royalty free test video',
1722                 'timestamp': 1432816365,
1723                 'upload_date': '20150528',
1724                 'is_live': False,
1725             },
1726             'params': {
1727                 'skip_download': True,
1728             },
1729             'add_ie': [ArkenaIE.ie_key()],
1730         },
1731         {
1732             'url': 'http://nova.bg/news/view/2016/08/16/156543/%D0%BD%D0%B0-%D0%BA%D0%BE%D1%81%D1%8A%D0%BC-%D0%BE%D1%82-%D0%B2%D0%B7%D1%80%D0%B8%D0%B2-%D0%BE%D1%82%D1%86%D0%B5%D0%BF%D0%B8%D1%85%D0%B0-%D1%86%D1%8F%D0%BB-%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB-%D0%B7%D0%B0%D1%80%D0%B0%D0%B4%D0%B8-%D0%B8%D0%B7%D1%82%D0%B8%D1%87%D0%B0%D0%BD%D0%B5-%D0%BD%D0%B0-%D0%B3%D0%B0%D0%B7-%D0%B2-%D0%BF%D0%BB%D0%BE%D0%B2%D0%B4%D0%B8%D0%B2/',
1733             'info_dict': {
1734                 'id': '1c7141f46c',
1735                 'ext': 'mp4',
1736                 'title': 'НА КОСЪМ ОТ ВЗРИВ: Изтичане на газ на бензиностанция в Пловдив',
1737             },
1738             'params': {
1739                 'skip_download': True,
1740             },
1741             'add_ie': [Vbox7IE.ie_key()],
1742         },
1743         {
1744             # DBTV embeds
1745             'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
1746             'info_dict': {
1747                 'id': '43254897',
1748                 'title': 'Etter ett års planlegging, klaffet endelig alt: - Jeg måtte ta en liten dans',
1749             },
1750             'playlist_mincount': 3,
1751         },
1752         {
1753             # Videa embeds
1754             'url': 'http://forum.dvdtalk.com/movie-talk/623756-deleted-magic-star-wars-ot-deleted-alt-scenes-docu-style.html',
1755             'info_dict': {
1756                 'id': '623756-deleted-magic-star-wars-ot-deleted-alt-scenes-docu-style',
1757                 'title': 'Deleted Magic - Star Wars: OT Deleted / Alt. Scenes Docu. Style - DVD Talk Forum',
1758             },
1759             'playlist_mincount': 2,
1760         },
1761         {
1762             # 20 minuten embed
1763             'url': 'http://www.20min.ch/schweiz/news/story/So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
1764             'info_dict': {
1765                 'id': '523629',
1766                 'ext': 'mp4',
1767                 'title': 'So kommen Sie bei Eis und Schnee sicher an',
1768                 'description': 'md5:117c212f64b25e3d95747e5276863f7d',
1769             },
1770             'params': {
1771                 'skip_download': True,
1772             },
1773             'add_ie': [TwentyMinutenIE.ie_key()],
1774         },
1775         {
1776             # VideoPress embed
1777             'url': 'https://en.support.wordpress.com/videopress/',
1778             'info_dict': {
1779                 'id': 'OcobLTqC',
1780                 'ext': 'm4v',
1781                 'title': 'IMG_5786',
1782                 'timestamp': 1435711927,
1783                 'upload_date': '20150701',
1784             },
1785             'params': {
1786                 'skip_download': True,
1787             },
1788             'add_ie': [VideoPressIE.ie_key()],
1789         },
1790         {
1791             # Rutube embed
1792             'url': 'http://magazzino.friday.ru/videos/vipuski/kazan-2',
1793             'info_dict': {
1794                 'id': '9b3d5bee0a8740bf70dfd29d3ea43541',
1795                 'ext': 'flv',
1796                 'title': 'Магаззино: Казань 2',
1797                 'description': 'md5:99bccdfac2269f0e8fdbc4bbc9db184a',
1798                 'uploader': 'Магаззино',
1799                 'upload_date': '20170228',
1800                 'uploader_id': '996642',
1801             },
1802             'params': {
1803                 'skip_download': True,
1804             },
1805             'add_ie': [RutubeIE.ie_key()],
1806         },
1807         {
1808             # ThePlatform embedded with whitespaces in URLs
1809             'url': 'http://www.golfchannel.com/topics/shows/golftalkcentral.htm',
1810             'only_matching': True,
1811         },
1812         {
1813             # Senate ISVP iframe https
1814             'url': 'https://www.hsgac.senate.gov/hearings/canadas-fast-track-refugee-plan-unanswered-questions-and-implications-for-us-national-security',
1815             'md5': 'fb8c70b0b515e5037981a2492099aab8',
1816             'info_dict': {
1817                 'id': 'govtaff020316',
1818                 'ext': 'mp4',
1819                 'title': 'Integrated Senate Video Player',
1820             },
1821             'add_ie': [SenateISVPIE.ie_key()],
1822         },
1823         {
1824             # Limelight embeds (1 channel embed + 4 media embeds)
1825             'url': 'http://www.sedona.com/FacilitatorTraining2017',
1826             'info_dict': {
1827                 'id': 'FacilitatorTraining2017',
1828                 'title': 'Facilitator Training 2017',
1829             },
1830             'playlist_mincount': 5,
1831         },
1832         {
1833             # Limelight embed (LimelightPlayerUtil.embed)
1834             'url': 'https://tv5.ca/videos?v=xuu8qowr291ri',
1835             'info_dict': {
1836                 'id': '95d035dc5c8a401588e9c0e6bd1e9c92',
1837                 'ext': 'mp4',
1838                 'title': '07448641',
1839                 'timestamp': 1499890639,
1840                 'upload_date': '20170712',
1841             },
1842             'params': {
1843                 'skip_download': True,
1844             },
1845             'add_ie': ['LimelightMedia'],
1846         },
1847         {
1848             'url': 'http://kron4.com/2017/04/28/standoff-with-walnut-creek-murder-suspect-ends-with-arrest/',
1849             'info_dict': {
1850                 'id': 'standoff-with-walnut-creek-murder-suspect-ends-with-arrest',
1851                 'title': 'Standoff with Walnut Creek murder suspect ends',
1852                 'description': 'md5:3ccc48a60fc9441eeccfc9c469ebf788',
1853             },
1854             'playlist_mincount': 4,
1855         },
1856         {
1857             # WashingtonPost embed
1858             'url': 'http://www.vanityfair.com/hollywood/2017/04/donald-trump-tv-pitches',
1859             'info_dict': {
1860                 'id': '8caf6e88-d0ec-11e5-90d3-34c2c42653ac',
1861                 'ext': 'mp4',
1862                 'title': "No one has seen the drama series based on Trump's life \u2014 until now",
1863                 'description': 'Donald Trump wanted a weekly TV drama based on his life. It never aired. But The Washington Post recently obtained a scene from the pilot script — and enlisted actors.',
1864                 'timestamp': 1455216756,
1865                 'uploader': 'The Washington Post',
1866                 'upload_date': '20160211',
1867             },
1868             'add_ie': [WashingtonPostIE.ie_key()],
1869         },
1870         {
1871             # Mediaset embed
1872             'url': 'http://www.tgcom24.mediaset.it/politica/serracchiani-voglio-vivere-in-una-societa-aperta-reazioni-sproporzionate-_3071354-201702a.shtml',
1873             'info_dict': {
1874                 'id': '720642',
1875                 'ext': 'mp4',
1876                 'title': 'Serracchiani: "Voglio vivere in una società aperta, con tutela del patto di fiducia"',
1877             },
1878             'params': {
1879                 'skip_download': True,
1880             },
1881             'add_ie': [MediasetIE.ie_key()],
1882         },
1883         {
1884             # JOJ.sk embeds
1885             'url': 'https://www.noviny.sk/slovensko/238543-slovenskom-sa-prehnala-vlna-silnych-burok',
1886             'info_dict': {
1887                 'id': '238543-slovenskom-sa-prehnala-vlna-silnych-burok',
1888                 'title': 'Slovenskom sa prehnala vlna silných búrok',
1889             },
1890             'playlist_mincount': 5,
1891             'add_ie': [JojIE.ie_key()],
1892         },
1893         {
1894             # AMP embed (see https://www.ampproject.org/docs/reference/components/amp-video)
1895             'url': 'https://tvrain.ru/amp/418921/',
1896             'md5': 'cc00413936695987e8de148b67d14f1d',
1897             'info_dict': {
1898                 'id': '418921',
1899                 'ext': 'mp4',
1900                 'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
1901             },
1902         },
1903         {
1904             # vzaar embed
1905             'url': 'http://help.vzaar.com/article/165-embedding-video',
1906             'md5': '7e3919d9d2620b89e3e00bec7fe8c9d4',
1907             'info_dict': {
1908                 'id': '8707641',
1909                 'ext': 'mp4',
1910                 'title': 'Building A Business Online: Principal Chairs Q & A',
1911             },
1912         },
1913         {
1914             # multiple HTML5 videos on one page
1915             'url': 'https://www.paragon-software.com/home/rk-free/keyscenarios.html',
1916             'info_dict': {
1917                 'id': 'keyscenarios',
1918                 'title': 'Rescue Kit 14 Free Edition - Getting started',
1919             },
1920             'playlist_count': 4,
1921         },
1922         {
1923             # vshare embed
1924             'url': 'https://youtube-dl-demo.neocities.org/vshare.html',
1925             'md5': '17b39f55b5497ae8b59f5fbce8e35886',
1926             'info_dict': {
1927                 'id': '0f64ce6',
1928                 'title': 'vl14062007715967',
1929                 'ext': 'mp4',
1930             }
1931         },
1932         {
1933             'url': 'http://www.heidelberg-laureate-forum.org/blog/video/lecture-friday-september-23-2016-sir-c-antony-r-hoare/',
1934             'md5': 'aecd089f55b1cb5a59032cb049d3a356',
1935             'info_dict': {
1936                 'id': '90227f51a80c4d8f86c345a7fa62bd9a1d',
1937                 'ext': 'mp4',
1938                 'title': 'Lecture: Friday, September 23, 2016 - Sir Tony Hoare',
1939                 'description': 'md5:5a51db84a62def7b7054df2ade403c6c',
1940                 'timestamp': 1474354800,
1941                 'upload_date': '20160920',
1942             }
1943         },
1944         {
1945             'url': 'http://www.kidzworld.com/article/30935-trolls-the-beat-goes-on-interview-skylar-astin-and-amanda-leighton',
1946             'info_dict': {
1947                 'id': '1731611',
1948                 'ext': 'mp4',
1949                 'title': 'Official Trailer | TROLLS: THE BEAT GOES ON!',
1950                 'description': 'md5:eb5f23826a027ba95277d105f248b825',
1951                 'timestamp': 1516100691,
1952                 'upload_date': '20180116',
1953             },
1954             'params': {
1955                 'skip_download': True,
1956             },
1957             'add_ie': [SpringboardPlatformIE.ie_key()],
1958         },
1959         {
1960             'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU',
1961             'info_dict': {
1962                 'id': 'uPDB5I9wfp8',
1963                 'ext': 'webm',
1964                 'title': 'Pocoyo: 90 minutos de episódios completos Português para crianças - PARTE 3',
1965                 'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d',
1966                 'upload_date': '20160219',
1967                 'uploader': 'Pocoyo - Português (BR)',
1968                 'uploader_id': 'PocoyoBrazil',
1969             },
1970             'add_ie': [YoutubeIE.ie_key()],
1971             'params': {
1972                 'skip_download': True,
1973             },
1974         },
1975         {
1976             'url': 'https://www.yapfiles.ru/show/1872528/690b05d3054d2dbe1e69523aa21bb3b1.mp4.html',
1977             'info_dict': {
1978                 'id': 'vMDE4NzI1Mjgt690b',
1979                 'ext': 'mp4',
1980                 'title': 'Котята',
1981             },
1982             'add_ie': [YapFilesIE.ie_key()],
1983             'params': {
1984                 'skip_download': True,
1985             },
1986         }
1987         # {
1988         #     # TODO: find another test
1989         #     # http://schema.org/VideoObject
1990         #     'url': 'https://flipagram.com/f/nyvTSJMKId',
1991         #     'md5': '888dcf08b7ea671381f00fab74692755',
1992         #     'info_dict': {
1993         #         'id': 'nyvTSJMKId',
1994         #         'ext': 'mp4',
1995         #         'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
1996         #         'description': '#love for cats.',
1997         #         'timestamp': 1461244995,
1998         #         'upload_date': '20160421',
1999         #     },
2000         #     'params': {
2001         #         'force_generic_extractor': True,
2002         #     },
2003         # }
2004     ]
2005
2006     def report_following_redirect(self, new_url):
2007         """Report information extraction."""
2008         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
2009
2010     def _extract_rss(self, url, video_id, doc):
2011         playlist_title = doc.find('./channel/title').text
2012         playlist_desc_el = doc.find('./channel/description')
2013         playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
2014
2015         entries = []
2016         for it in doc.findall('./channel/item'):
2017             next_url = xpath_text(it, 'link', fatal=False)
2018             if not next_url:
2019                 enclosure_nodes = it.findall('./enclosure')
2020                 for e in enclosure_nodes:
2021                     next_url = e.attrib.get('url')
2022                     if next_url:
2023                         break
2024
2025             if not next_url:
2026                 continue
2027
2028             entries.append({
2029                 '_type': 'url_transparent',
2030                 'url': next_url,
2031                 'title': it.find('title').text,
2032             })
2033
2034         return {
2035             '_type': 'playlist',
2036             'id': url,
2037             'title': playlist_title,
2038             'description': playlist_desc,
2039             'entries': entries,
2040         }
2041
2042     def _extract_camtasia(self, url, video_id, webpage):
2043         """ Returns None if no camtasia video can be found. """
2044
2045         camtasia_cfg = self._search_regex(
2046             r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
2047             webpage, 'camtasia configuration file', default=None)
2048         if camtasia_cfg is None:
2049             return None
2050
2051         title = self._html_search_meta('DC.title', webpage, fatal=True)
2052
2053         camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
2054         camtasia_cfg = self._download_xml(
2055             camtasia_url, video_id,
2056             note='Downloading camtasia configuration',
2057             errnote='Failed to download camtasia configuration')
2058         fileset_node = camtasia_cfg.find('./playlist/array/fileset')
2059
2060         entries = []
2061         for n in fileset_node.getchildren():
2062             url_n = n.find('./uri')
2063             if url_n is None:
2064                 continue
2065
2066             entries.append({
2067                 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
2068                 'title': '%s - %s' % (title, n.tag),
2069                 'url': compat_urlparse.urljoin(url, url_n.text),
2070                 'duration': float_or_none(n.find('./duration').text),
2071             })
2072
2073         return {
2074             '_type': 'playlist',
2075             'entries': entries,
2076             'title': title,
2077         }
2078
2079     def _real_extract(self, url):
2080         if url.startswith('//'):
2081             return {
2082                 '_type': 'url',
2083                 'url': self.http_scheme() + url,
2084             }
2085
2086         parsed_url = compat_urlparse.urlparse(url)
2087         if not parsed_url.scheme:
2088             default_search = self._downloader.params.get('default_search')
2089             if default_search is None:
2090                 default_search = 'fixup_error'
2091
2092             if default_search in ('auto', 'auto_warning', 'fixup_error'):
2093                 if '/' in url:
2094                     self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
2095                     return self.url_result('http://' + url)
2096                 elif default_search != 'fixup_error':
2097                     if default_search == 'auto_warning':
2098                         if re.match(r'^(?:url|URL)$', url):
2099                             raise ExtractorError(
2100                                 'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
2101                                 expected=True)
2102                         else:
2103                             self._downloader.report_warning(
2104                                 'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
2105                     return self.url_result('ytsearch:' + url)
2106
2107             if default_search in ('error', 'fixup_error'):
2108                 raise ExtractorError(
2109                     '%r is not a valid URL. '
2110                     'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
2111                     % (url, url), expected=True)
2112             else:
2113                 if ':' not in default_search:
2114                     default_search += ':'
2115                 return self.url_result(default_search + url)
2116
2117         url, smuggled_data = unsmuggle_url(url)
2118         force_videoid = None
2119         is_intentional = smuggled_data and smuggled_data.get('to_generic')
2120         if smuggled_data and 'force_videoid' in smuggled_data:
2121             force_videoid = smuggled_data['force_videoid']
2122             video_id = force_videoid
2123         else:
2124             video_id = self._generic_id(url)
2125
2126         self.to_screen('%s: Requesting header' % video_id)
2127
2128         head_req = HEADRequest(url)
2129         head_response = self._request_webpage(
2130             head_req, video_id,
2131             note=False, errnote='Could not send HEAD request to %s' % url,
2132             fatal=False)
2133
2134         if head_response is not False:
2135             # Check for redirect
2136             new_url = compat_str(head_response.geturl())
2137             if url != new_url:
2138                 self.report_following_redirect(new_url)
2139                 if force_videoid:
2140                     new_url = smuggle_url(
2141                         new_url, {'force_videoid': force_videoid})
2142                 return self.url_result(new_url)
2143
2144         full_response = None
2145         if head_response is False:
2146             request = sanitized_Request(url)
2147             request.add_header('Accept-Encoding', '*')
2148             full_response = self._request_webpage(request, video_id)
2149             head_response = full_response
2150
2151         info_dict = {
2152             'id': video_id,
2153             'title': self._generic_title(url),
2154             'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
2155         }
2156
2157         # Check for direct link to a video
2158         content_type = head_response.headers.get('Content-Type', '').lower()
2159         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
2160         if m:
2161             format_id = compat_str(m.group('format_id'))
2162             if format_id.endswith('mpegurl'):
2163                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
2164             elif format_id == 'f4m':
2165                 formats = self._extract_f4m_formats(url, video_id)
2166             else:
2167                 formats = [{
2168                     'format_id': format_id,
2169                     'url': url,
2170                     'vcodec': 'none' if m.group('type') == 'audio' else None
2171                 }]
2172                 info_dict['direct'] = True
2173             self._sort_formats(formats)
2174             info_dict['formats'] = formats
2175             return info_dict
2176
2177         if not self._downloader.params.get('test', False) and not is_intentional:
2178             force = self._downloader.params.get('force_generic_extractor', False)
2179             self._downloader.report_warning(
2180                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
2181
2182         if not full_response:
2183             request = sanitized_Request(url)
2184             # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
2185             # making it impossible to download only chunk of the file (yet we need only 512kB to
2186             # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
2187             # that will always result in downloading the whole file that is not desirable.
2188             # Therefore for extraction pass we have to override Accept-Encoding to any in order
2189             # to accept raw bytes and being able to download only a chunk.
2190             # It may probably better to solve this by checking Content-Type for application/octet-stream
2191             # after HEAD request finishes, but not sure if we can rely on this.
2192             request.add_header('Accept-Encoding', '*')
2193             full_response = self._request_webpage(request, video_id)
2194
2195         first_bytes = full_response.read(512)
2196
2197         # Is it an M3U playlist?
2198         if first_bytes.startswith(b'#EXTM3U'):
2199             info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
2200             self._sort_formats(info_dict['formats'])
2201             return info_dict
2202
2203         # Maybe it's a direct link to a video?
2204         # Be careful not to download the whole thing!
2205         if not is_html(first_bytes):
2206             self._downloader.report_warning(
2207                 'URL could be a direct video link, returning it as such.')
2208             info_dict.update({
2209                 'direct': True,
2210                 'url': url,
2211             })
2212             return info_dict
2213
2214         webpage = self._webpage_read_content(
2215             full_response, url, video_id, prefix=first_bytes)
2216
2217         self.report_extraction(video_id)
2218
2219         # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
2220         try:
2221             doc = compat_etree_fromstring(webpage.encode('utf-8'))
2222             if doc.tag == 'rss':
2223                 return self._extract_rss(url, video_id, doc)
2224             elif doc.tag == 'SmoothStreamingMedia':
2225                 info_dict['formats'] = self._parse_ism_formats(doc, url)
2226                 self._sort_formats(info_dict['formats'])
2227                 return info_dict
2228             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
2229                 smil = self._parse_smil(doc, url, video_id)
2230                 self._sort_formats(smil['formats'])
2231                 return smil
2232             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
2233                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
2234             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
2235                 info_dict['formats'] = self._parse_mpd_formats(
2236                     doc,
2237                     mpd_base_url=compat_str(full_response.geturl()).rpartition('/')[0],
2238                     mpd_url=url)
2239                 self._sort_formats(info_dict['formats'])
2240                 return info_dict
2241             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
2242                 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
2243                 self._sort_formats(info_dict['formats'])
2244                 return info_dict
2245         except compat_xml_parse_error:
2246             pass
2247
2248         # Is it a Camtasia project?
2249         camtasia_res = self._extract_camtasia(url, video_id, webpage)
2250         if camtasia_res is not None:
2251             return camtasia_res
2252
2253         # Sometimes embedded video player is hidden behind percent encoding
2254         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
2255         # Unescaping the whole page allows to handle those cases in a generic way
2256         webpage = compat_urllib_parse_unquote(webpage)
2257
2258         # it's tempting to parse this further, but you would
2259         # have to take into account all the variations like
2260         #   Video Title - Site Name
2261         #   Site Name | Video Title
2262         #   Video Title - Tagline | Site Name
2263         # and so on and so forth; it's just not practical
2264         video_title = self._og_search_title(
2265             webpage, default=None) or self._html_search_regex(
2266             r'(?s)<title>(.*?)</title>', webpage, 'video title',
2267             default='video')
2268
2269         # Try to detect age limit automatically
2270         age_limit = self._rta_search(webpage)
2271         # And then there are the jokers who advertise that they use RTA,
2272         # but actually don't.
2273         AGE_LIMIT_MARKERS = [
2274             r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
2275         ]
2276         if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
2277             age_limit = 18
2278
2279         # video uploader is domain name
2280         video_uploader = self._search_regex(
2281             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
2282
2283         video_description = self._og_search_description(webpage, default=None)
2284         video_thumbnail = self._og_search_thumbnail(webpage, default=None)
2285
2286         info_dict.update({
2287             'title': video_title,
2288             'description': video_description,
2289             'thumbnail': video_thumbnail,
2290             'age_limit': age_limit,
2291         })
2292
2293         # Look for Brightcove Legacy Studio embeds
2294         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
2295         if bc_urls:
2296             entries = [{
2297                 '_type': 'url',
2298                 'url': smuggle_url(bc_url, {'Referer': url}),
2299                 'ie_key': 'BrightcoveLegacy'
2300             } for bc_url in bc_urls]
2301
2302             return {
2303                 '_type': 'playlist',
2304                 'title': video_title,
2305                 'id': video_id,
2306                 'entries': entries,
2307             }
2308
2309         # Look for Brightcove New Studio embeds
2310         bc_urls = BrightcoveNewIE._extract_urls(self, webpage)
2311         if bc_urls:
2312             return self.playlist_from_matches(
2313                 bc_urls, video_id, video_title,
2314                 getter=lambda x: smuggle_url(x, {'referrer': url}),
2315                 ie='BrightcoveNew')
2316
2317         # Look for Nexx embeds
2318         nexx_urls = NexxIE._extract_urls(webpage)
2319         if nexx_urls:
2320             return self.playlist_from_matches(nexx_urls, video_id, video_title, ie=NexxIE.ie_key())
2321
2322         # Look for Nexx iFrame embeds
2323         nexx_embed_urls = NexxEmbedIE._extract_urls(webpage)
2324         if nexx_embed_urls:
2325             return self.playlist_from_matches(nexx_embed_urls, video_id, video_title, ie=NexxEmbedIE.ie_key())
2326
2327         # Look for ThePlatform embeds
2328         tp_urls = ThePlatformIE._extract_urls(webpage)
2329         if tp_urls:
2330             return self.playlist_from_matches(tp_urls, video_id, video_title, ie='ThePlatform')
2331
2332         # Look for Vessel embeds
2333         vessel_urls = VesselIE._extract_urls(webpage)
2334         if vessel_urls:
2335             return self.playlist_from_matches(vessel_urls, video_id, video_title, ie=VesselIE.ie_key())
2336
2337         # Look for embedded rtl.nl player
2338         matches = re.findall(
2339             r'<iframe[^>]+?src="((?:https?:)?//(?:(?:www|static)\.)?rtl\.nl/(?:system/videoplayer/[^"]+(?:video_)?)?embed[^"]+)"',
2340             webpage)
2341         if matches:
2342             return self.playlist_from_matches(matches, video_id, video_title, ie='RtlNl')
2343
2344         vimeo_urls = VimeoIE._extract_urls(url, webpage)
2345         if vimeo_urls:
2346             return self.playlist_from_matches(vimeo_urls, video_id, video_title, ie=VimeoIE.ie_key())
2347
2348         vid_me_embed_url = self._search_regex(
2349             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
2350             webpage, 'vid.me embed', default=None)
2351         if vid_me_embed_url is not None:
2352             return self.url_result(vid_me_embed_url, 'Vidme')
2353
2354         # Look for YouTube embeds
2355         youtube_urls = YoutubeIE._extract_urls(webpage)
2356         if youtube_urls:
2357             return self.playlist_from_matches(
2358                 youtube_urls, video_id, video_title, ie=YoutubeIE.ie_key())
2359
2360         matches = DailymotionIE._extract_urls(webpage)
2361         if matches:
2362             return self.playlist_from_matches(matches, video_id, video_title)
2363
2364         # Look for embedded Dailymotion playlist player (#3822)
2365         m = re.search(
2366             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
2367         if m:
2368             playlists = re.findall(
2369                 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
2370             if playlists:
2371                 return self.playlist_from_matches(
2372                     playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
2373
2374         # Look for DailyMail embeds
2375         dailymail_urls = DailyMailIE._extract_urls(webpage)
2376         if dailymail_urls:
2377             return self.playlist_from_matches(
2378                 dailymail_urls, video_id, video_title, ie=DailyMailIE.ie_key())
2379
2380         # Look for embedded Wistia player
2381         wistia_url = WistiaIE._extract_url(webpage)
2382         if wistia_url:
2383             return {
2384                 '_type': 'url_transparent',
2385                 'url': self._proto_relative_url(wistia_url),
2386                 'ie_key': WistiaIE.ie_key(),
2387                 'uploader': video_uploader,
2388             }
2389
2390         # Look for SVT player
2391         svt_url = SVTIE._extract_url(webpage)
2392         if svt_url:
2393             return self.url_result(svt_url, 'SVT')
2394
2395         # Look for Bandcamp pages with custom domain
2396         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
2397         if mobj is not None:
2398             burl = unescapeHTML(mobj.group(1))
2399             # Don't set the extractor because it can be a track url or an album
2400             return self.url_result(burl)
2401
2402         # Look for embedded Vevo player
2403         mobj = re.search(
2404             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
2405         if mobj is not None:
2406             return self.url_result(mobj.group('url'))
2407
2408         # Look for embedded Viddler player
2409         mobj = re.search(
2410             r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
2411             webpage)
2412         if mobj is not None:
2413             return self.url_result(mobj.group('url'))
2414
2415         # Look for NYTimes player
2416         mobj = re.search(
2417             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
2418             webpage)
2419         if mobj is not None:
2420             return self.url_result(mobj.group('url'))
2421
2422         # Look for Libsyn player
2423         mobj = re.search(
2424             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
2425         if mobj is not None:
2426             return self.url_result(mobj.group('url'))
2427
2428         # Look for Ooyala videos
2429         mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
2430                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
2431                 re.search(r'OO\.Player\.create\.apply\(\s*OO\.Player\s*,\s*op\(\s*\[\s*[\'"][^\'"]*[\'"]\s*,\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
2432                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
2433                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
2434         if mobj is not None:
2435             embed_token = self._search_regex(
2436                 r'embedToken[\'"]?\s*:\s*[\'"]([^\'"]+)',
2437                 webpage, 'ooyala embed token', default=None)
2438             return OoyalaIE._build_url_result(smuggle_url(
2439                 mobj.group('ec'), {
2440                     'domain': url,
2441                     'embed_token': embed_token,
2442                 }))
2443
2444         # Look for multiple Ooyala embeds on SBN network websites
2445         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
2446         if mobj is not None:
2447             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
2448             if embeds:
2449                 return self.playlist_from_matches(
2450                     embeds, video_id, video_title,
2451                     getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
2452
2453         # Look for Aparat videos
2454         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
2455         if mobj is not None:
2456             return self.url_result(mobj.group(1), 'Aparat')
2457
2458         # Look for MPORA videos
2459         mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
2460         if mobj is not None:
2461             return self.url_result(mobj.group(1), 'Mpora')
2462
2463         # Look for embedded NovaMov-based player
2464         mobj = re.search(
2465             r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
2466                     (?P<url>http://(?:(?:embed|www)\.)?
2467                         (?:novamov\.com|
2468                            nowvideo\.(?:ch|sx|eu|at|ag|co)|
2469                            videoweed\.(?:es|com)|
2470                            movshare\.(?:net|sx|ag)|
2471                            divxstage\.(?:eu|net|ch|co|at|ag))
2472                         /embed\.php.+?)\1''', webpage)
2473         if mobj is not None:
2474             return self.url_result(mobj.group('url'))
2475
2476         # Look for embedded Facebook player
2477         facebook_urls = FacebookIE._extract_urls(webpage)
2478         if facebook_urls:
2479             return self.playlist_from_matches(facebook_urls, video_id, video_title)
2480
2481         # Look for embedded VK player
2482         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
2483         if mobj is not None:
2484             return self.url_result(mobj.group('url'), 'VK')
2485
2486         # Look for embedded Odnoklassniki player
2487         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
2488         if mobj is not None:
2489             return self.url_result(mobj.group('url'), 'Odnoklassniki')
2490
2491         # Look for embedded ivi player
2492         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
2493         if mobj is not None:
2494             return self.url_result(mobj.group('url'), 'Ivi')
2495
2496         # Look for embedded Huffington Post player
2497         mobj = re.search(
2498             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
2499         if mobj is not None:
2500             return self.url_result(mobj.group('url'), 'HuffPost')
2501
2502         # Look for embed.ly
2503         mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
2504         if mobj is not None:
2505             return self.url_result(mobj.group('url'))
2506         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
2507         if mobj is not None:
2508             return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
2509
2510         # Look for funnyordie embed
2511         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
2512         if matches:
2513             return self.playlist_from_matches(
2514                 matches, video_id, video_title, getter=unescapeHTML, ie='FunnyOrDie')
2515
2516         # Look for BBC iPlayer embed
2517         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
2518         if matches:
2519             return self.playlist_from_matches(matches, video_id, video_title, ie='BBCCoUk')
2520
2521         # Look for embedded RUTV player
2522         rutv_url = RUTVIE._extract_url(webpage)
2523         if rutv_url:
2524             return self.url_result(rutv_url, 'RUTV')
2525
2526         # Look for embedded TVC player
2527         tvc_url = TVCIE._extract_url(webpage)
2528         if tvc_url:
2529             return self.url_result(tvc_url, 'TVC')
2530
2531         # Look for embedded SportBox player
2532         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
2533         if sportbox_urls:
2534             return self.playlist_from_matches(sportbox_urls, video_id, video_title, ie='SportBoxEmbed')
2535
2536         # Look for embedded XHamster player
2537         xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
2538         if xhamster_urls:
2539             return self.playlist_from_matches(xhamster_urls, video_id, video_title, ie='XHamsterEmbed')
2540
2541         # Look for embedded TNAFlixNetwork player
2542         tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
2543         if tnaflix_urls:
2544             return self.playlist_from_matches(tnaflix_urls, video_id, video_title, ie=TNAFlixNetworkEmbedIE.ie_key())
2545
2546         # Look for embedded PornHub player
2547         pornhub_urls = PornHubIE._extract_urls(webpage)
2548         if pornhub_urls:
2549             return self.playlist_from_matches(pornhub_urls, video_id, video_title, ie=PornHubIE.ie_key())
2550
2551         # Look for embedded DrTuber player
2552         drtuber_urls = DrTuberIE._extract_urls(webpage)
2553         if drtuber_urls:
2554             return self.playlist_from_matches(drtuber_urls, video_id, video_title, ie=DrTuberIE.ie_key())
2555
2556         # Look for embedded RedTube player
2557         redtube_urls = RedTubeIE._extract_urls(webpage)
2558         if redtube_urls:
2559             return self.playlist_from_matches(redtube_urls, video_id, video_title, ie=RedTubeIE.ie_key())
2560
2561         # Look for embedded Tvigle player
2562         mobj = re.search(
2563             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
2564         if mobj is not None:
2565             return self.url_result(mobj.group('url'), 'Tvigle')
2566
2567         # Look for embedded TED player
2568         mobj = re.search(
2569             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
2570         if mobj is not None:
2571             return self.url_result(mobj.group('url'), 'TED')
2572
2573         # Look for embedded Ustream videos
2574         ustream_url = UstreamIE._extract_url(webpage)
2575         if ustream_url:
2576             return self.url_result(ustream_url, UstreamIE.ie_key())
2577
2578         # Look for embedded arte.tv player
2579         mobj = re.search(
2580             r'<(?:script|iframe) [^>]*?src="(?P<url>http://www\.arte\.tv/(?:playerv2/embed|arte_vp/index)[^"]+)"',
2581             webpage)
2582         if mobj is not None:
2583             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
2584
2585         # Look for embedded francetv player
2586         mobj = re.search(
2587             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
2588             webpage)
2589         if mobj is not None:
2590             return self.url_result(mobj.group('url'))
2591
2592         # Look for embedded smotri.com player
2593         smotri_url = SmotriIE._extract_url(webpage)
2594         if smotri_url:
2595             return self.url_result(smotri_url, 'Smotri')
2596
2597         # Look for embedded Myvi.ru player
2598         myvi_url = MyviIE._extract_url(webpage)
2599         if myvi_url:
2600             return self.url_result(myvi_url)
2601
2602         # Look for embedded soundcloud player
2603         soundcloud_urls = SoundcloudIE._extract_urls(webpage)
2604         if soundcloud_urls:
2605             return self.playlist_from_matches(soundcloud_urls, video_id, video_title, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
2606
2607         # Look for tunein player
2608         tunein_urls = TuneInBaseIE._extract_urls(webpage)
2609         if tunein_urls:
2610             return self.playlist_from_matches(tunein_urls, video_id, video_title)
2611
2612         # Look for embedded mtvservices player
2613         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
2614         if mtvservices_url:
2615             return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
2616
2617         # Look for embedded yahoo player
2618         mobj = re.search(
2619             r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
2620             webpage)
2621         if mobj is not None:
2622             return self.url_result(mobj.group('url'), 'Yahoo')
2623
2624         # Look for embedded sbs.com.au player
2625         mobj = re.search(
2626             r'''(?x)
2627             (?:
2628                 <meta\s+property="og:video"\s+content=|
2629                 <iframe[^>]+?src=
2630             )
2631             (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
2632             webpage)
2633         if mobj is not None:
2634             return self.url_result(mobj.group('url'), 'SBS')
2635
2636         # Look for embedded Cinchcast player
2637         mobj = re.search(
2638             r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
2639             webpage)
2640         if mobj is not None:
2641             return self.url_result(mobj.group('url'), 'Cinchcast')
2642
2643         mobj = re.search(
2644             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
2645             webpage)
2646         if not mobj:
2647             mobj = re.search(
2648                 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
2649                 webpage)
2650         if mobj is not None:
2651             return self.url_result(mobj.group('url'), 'MLB')
2652
2653         mobj = re.search(
2654             r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
2655             webpage)
2656         if mobj is not None:
2657             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
2658
2659         mobj = re.search(
2660             r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"',
2661             webpage)
2662         if mobj is not None:
2663             return self.url_result(mobj.group('url'), 'Livestream')
2664
2665         # Look for Zapiks embed
2666         mobj = re.search(
2667             r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
2668         if mobj is not None:
2669             return self.url_result(mobj.group('url'), 'Zapiks')
2670
2671         # Look for Kaltura embeds
2672         kaltura_url = KalturaIE._extract_url(webpage)
2673         if kaltura_url:
2674             return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
2675
2676         # Look for EaglePlatform embeds
2677         eagleplatform_url = EaglePlatformIE._extract_url(webpage)
2678         if eagleplatform_url:
2679             return self.url_result(smuggle_url(eagleplatform_url, {'referrer': url}), EaglePlatformIE.ie_key())
2680
2681         # Look for ClipYou (uses EaglePlatform) embeds
2682         mobj = re.search(
2683             r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
2684         if mobj is not None:
2685             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
2686
2687         # Look for Pladform embeds
2688         pladform_url = PladformIE._extract_url(webpage)
2689         if pladform_url:
2690             return self.url_result(pladform_url)
2691
2692         # Look for Videomore embeds
2693         videomore_url = VideomoreIE._extract_url(webpage)
2694         if videomore_url:
2695             return self.url_result(videomore_url)
2696
2697         # Look for Webcaster embeds
2698         webcaster_url = WebcasterFeedIE._extract_url(self, webpage)
2699         if webcaster_url:
2700             return self.url_result(webcaster_url, ie=WebcasterFeedIE.ie_key())
2701
2702         # Look for Playwire embeds
2703         mobj = re.search(
2704             r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
2705         if mobj is not None:
2706             return self.url_result(mobj.group('url'))
2707
2708         # Look for 5min embeds
2709         mobj = re.search(
2710             r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
2711         if mobj is not None:
2712             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
2713
2714         # Look for Crooks and Liars embeds
2715         mobj = re.search(
2716             r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
2717         if mobj is not None:
2718             return self.url_result(mobj.group('url'))
2719
2720         # Look for NBC Sports VPlayer embeds
2721         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
2722         if nbc_sports_url:
2723             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
2724
2725         # Look for NBC News embeds
2726         nbc_news_embed_url = re.search(
2727             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1', webpage)
2728         if nbc_news_embed_url:
2729             return self.url_result(nbc_news_embed_url.group('url'), 'NBCNews')
2730
2731         # Look for Google Drive embeds
2732         google_drive_url = GoogleDriveIE._extract_url(webpage)
2733         if google_drive_url:
2734             return self.url_result(google_drive_url, 'GoogleDrive')
2735
2736         # Look for UDN embeds
2737         mobj = re.search(
2738             r'<iframe[^>]+src="(?:https?:)?(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
2739         if mobj is not None:
2740             return self.url_result(
2741                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
2742
2743         # Look for Senate ISVP iframe
2744         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
2745         if senate_isvp_url:
2746             return self.url_result(senate_isvp_url, 'SenateISVP')
2747
2748         # Look for OnionStudios embeds
2749         onionstudios_url = OnionStudiosIE._extract_url(webpage)
2750         if onionstudios_url:
2751             return self.url_result(onionstudios_url)
2752
2753         # Look for ViewLift embeds
2754         viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
2755         if viewlift_url:
2756             return self.url_result(viewlift_url)
2757
2758         # Look for JWPlatform embeds
2759         jwplatform_urls = JWPlatformIE._extract_urls(webpage)
2760         if jwplatform_urls:
2761             return self.playlist_from_matches(jwplatform_urls, video_id, video_title, ie=JWPlatformIE.ie_key())
2762
2763         # Look for Digiteka embeds
2764         digiteka_url = DigitekaIE._extract_url(webpage)
2765         if digiteka_url:
2766             return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
2767
2768         # Look for Arkena embeds
2769         arkena_url = ArkenaIE._extract_url(webpage)
2770         if arkena_url:
2771             return self.url_result(arkena_url, ArkenaIE.ie_key())
2772
2773         # Look for Piksel embeds
2774         piksel_url = PikselIE._extract_url(webpage)
2775         if piksel_url:
2776             return self.url_result(piksel_url, PikselIE.ie_key())
2777
2778         # Look for Limelight embeds
2779         limelight_urls = LimelightBaseIE._extract_urls(webpage, url)
2780         if limelight_urls:
2781             return self.playlist_result(
2782                 limelight_urls, video_id, video_title, video_description)
2783
2784         # Look for Anvato embeds
2785         anvato_urls = AnvatoIE._extract_urls(self, webpage, video_id)
2786         if anvato_urls:
2787             return self.playlist_result(
2788                 anvato_urls, video_id, video_title, video_description)
2789
2790         # Look for AdobeTVVideo embeds
2791         mobj = re.search(
2792             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
2793             webpage)
2794         if mobj is not None:
2795             return self.url_result(
2796                 self._proto_relative_url(unescapeHTML(mobj.group(1))),
2797                 'AdobeTVVideo')
2798
2799         # Look for Vine embeds
2800         mobj = re.search(
2801             r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
2802             webpage)
2803         if mobj is not None:
2804             return self.url_result(
2805                 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
2806
2807         # Look for VODPlatform embeds
2808         mobj = re.search(
2809             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vod-platform\.net/[eE]mbed/.+?)\1',
2810             webpage)
2811         if mobj is not None:
2812             return self.url_result(
2813                 self._proto_relative_url(unescapeHTML(mobj.group('url'))), 'VODPlatform')
2814
2815         # Look for Mangomolo embeds
2816         mobj = re.search(
2817             r'''(?x)<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?admin\.mangomolo\.com/analytics/index\.php/customers/embed/
2818                 (?:
2819                     video\?.*?\bid=(?P<video_id>\d+)|
2820                     index\?.*?\bchannelid=(?P<channel_id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)
2821                 ).+?)\1''', webpage)
2822         if mobj is not None:
2823             info = {
2824                 '_type': 'url_transparent',
2825                 'url': self._proto_relative_url(unescapeHTML(mobj.group('url'))),
2826                 'title': video_title,
2827                 'description': video_description,
2828                 'thumbnail': video_thumbnail,
2829                 'uploader': video_uploader,
2830             }
2831             video_id = mobj.group('video_id')
2832             if video_id:
2833                 info.update({
2834                     'ie_key': 'MangomoloVideo',
2835                     'id': video_id,
2836                 })
2837             else:
2838                 info.update({
2839                     'ie_key': 'MangomoloLive',
2840                     'id': mobj.group('channel_id'),
2841                 })
2842             return info
2843
2844         # Look for Instagram embeds
2845         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
2846         if instagram_embed_url is not None:
2847             return self.url_result(
2848                 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
2849
2850         # Look for LiveLeak embeds
2851         liveleak_urls = LiveLeakIE._extract_urls(webpage)
2852         if liveleak_urls:
2853             return self.playlist_from_matches(liveleak_urls, video_id, video_title)
2854
2855         # Look for 3Q SDN embeds
2856         threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
2857         if threeqsdn_url:
2858             return {
2859                 '_type': 'url_transparent',
2860                 'ie_key': ThreeQSDNIE.ie_key(),
2861                 'url': self._proto_relative_url(threeqsdn_url),
2862                 'title': video_title,
2863                 'description': video_description,
2864                 'thumbnail': video_thumbnail,
2865                 'uploader': video_uploader,
2866             }
2867
2868         # Look for VBOX7 embeds
2869         vbox7_url = Vbox7IE._extract_url(webpage)
2870         if vbox7_url:
2871             return self.url_result(vbox7_url, Vbox7IE.ie_key())
2872
2873         # Look for DBTV embeds
2874         dbtv_urls = DBTVIE._extract_urls(webpage)
2875         if dbtv_urls:
2876             return self.playlist_from_matches(dbtv_urls, video_id, video_title, ie=DBTVIE.ie_key())
2877
2878         # Look for Videa embeds
2879         videa_urls = VideaIE._extract_urls(webpage)
2880         if videa_urls:
2881             return self.playlist_from_matches(videa_urls, video_id, video_title, ie=VideaIE.ie_key())
2882
2883         # Look for 20 minuten embeds
2884         twentymin_urls = TwentyMinutenIE._extract_urls(webpage)
2885         if twentymin_urls:
2886             return self.playlist_from_matches(
2887                 twentymin_urls, video_id, video_title, ie=TwentyMinutenIE.ie_key())
2888
2889         # Look for Openload embeds
2890         openload_urls = OpenloadIE._extract_urls(webpage)
2891         if openload_urls:
2892             return self.playlist_from_matches(
2893                 openload_urls, video_id, video_title, ie=OpenloadIE.ie_key())
2894
2895         # Look for VideoPress embeds
2896         videopress_urls = VideoPressIE._extract_urls(webpage)
2897         if videopress_urls:
2898             return self.playlist_from_matches(
2899                 videopress_urls, video_id, video_title, ie=VideoPressIE.ie_key())
2900
2901         # Look for Rutube embeds
2902         rutube_urls = RutubeIE._extract_urls(webpage)
2903         if rutube_urls:
2904             return self.playlist_from_matches(
2905                 rutube_urls, video_id, video_title, ie=RutubeIE.ie_key())
2906
2907         # Look for WashingtonPost embeds
2908         wapo_urls = WashingtonPostIE._extract_urls(webpage)
2909         if wapo_urls:
2910             return self.playlist_from_matches(
2911                 wapo_urls, video_id, video_title, ie=WashingtonPostIE.ie_key())
2912
2913         # Look for Mediaset embeds
2914         mediaset_urls = MediasetIE._extract_urls(webpage)
2915         if mediaset_urls:
2916             return self.playlist_from_matches(
2917                 mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
2918
2919         # Look for JOJ.sk embeds
2920         joj_urls = JojIE._extract_urls(webpage)
2921         if joj_urls:
2922             return self.playlist_from_matches(
2923                 joj_urls, video_id, video_title, ie=JojIE.ie_key())
2924
2925         # Look for megaphone.fm embeds
2926         mpfn_urls = MegaphoneIE._extract_urls(webpage)
2927         if mpfn_urls:
2928             return self.playlist_from_matches(
2929                 mpfn_urls, video_id, video_title, ie=MegaphoneIE.ie_key())
2930
2931         # Look for vzaar embeds
2932         vzaar_urls = VzaarIE._extract_urls(webpage)
2933         if vzaar_urls:
2934             return self.playlist_from_matches(
2935                 vzaar_urls, video_id, video_title, ie=VzaarIE.ie_key())
2936
2937         channel9_urls = Channel9IE._extract_urls(webpage)
2938         if channel9_urls:
2939             return self.playlist_from_matches(
2940                 channel9_urls, video_id, video_title, ie=Channel9IE.ie_key())
2941
2942         vshare_urls = VShareIE._extract_urls(webpage)
2943         if vshare_urls:
2944             return self.playlist_from_matches(
2945                 vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
2946
2947         # Look for Mediasite embeds
2948         mediasite_urls = MediasiteIE._extract_urls(webpage)
2949         if mediasite_urls:
2950             entries = [
2951                 self.url_result(smuggle_url(
2952                     compat_urlparse.urljoin(url, mediasite_url),
2953                     {'UrlReferrer': url}), ie=MediasiteIE.ie_key())
2954                 for mediasite_url in mediasite_urls]
2955             return self.playlist_result(entries, video_id, video_title)
2956
2957         springboardplatform_urls = SpringboardPlatformIE._extract_urls(webpage)
2958         if springboardplatform_urls:
2959             return self.playlist_from_matches(
2960                 springboardplatform_urls, video_id, video_title,
2961                 ie=SpringboardPlatformIE.ie_key())
2962
2963         yapfiles_urls = YapFilesIE._extract_urls(webpage)
2964         if yapfiles_urls:
2965             return self.playlist_from_matches(
2966                 yapfiles_urls, video_id, video_title, ie=YapFilesIE.ie_key())
2967
2968         def merge_dicts(dict1, dict2):
2969             merged = {}
2970             for k, v in dict1.items():
2971                 if v is not None:
2972                     merged[k] = v
2973             for k, v in dict2.items():
2974                 if v is None:
2975                     continue
2976                 if (k not in merged or
2977                         (isinstance(v, compat_str) and v and
2978                             isinstance(merged[k], compat_str) and
2979                             not merged[k])):
2980                     merged[k] = v
2981             return merged
2982
2983         # Look for HTML5 media
2984         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
2985         if entries:
2986             if len(entries) == 1:
2987                 entries[0].update({
2988                     'id': video_id,
2989                     'title': video_title,
2990                 })
2991             else:
2992                 for num, entry in enumerate(entries, start=1):
2993                     entry.update({
2994                         'id': '%s-%s' % (video_id, num),
2995                         'title': '%s (%d)' % (video_title, num),
2996                     })
2997             for entry in entries:
2998                 self._sort_formats(entry['formats'])
2999             return self.playlist_result(entries, video_id, video_title)
3000
3001         jwplayer_data = self._find_jwplayer_data(
3002             webpage, video_id, transform_source=js_to_json)
3003         if jwplayer_data:
3004             info = self._parse_jwplayer_data(
3005                 jwplayer_data, video_id, require_title=False, base_url=url)
3006             return merge_dicts(info, info_dict)
3007
3008         # Video.js embed
3009         mobj = re.search(
3010             r'(?s)\bvideojs\s*\(.+?\.src\s*\(\s*((?:\[.+?\]|{.+?}))\s*\)\s*;',
3011             webpage)
3012         if mobj is not None:
3013             sources = self._parse_json(
3014                 mobj.group(1), video_id, transform_source=js_to_json,
3015                 fatal=False) or []
3016             if not isinstance(sources, list):
3017                 sources = [sources]
3018             formats = []
3019             for source in sources:
3020                 src = source.get('src')
3021                 if not src or not isinstance(src, compat_str):
3022                     continue
3023                 src = compat_urlparse.urljoin(url, src)
3024                 src_type = source.get('type')
3025                 if isinstance(src_type, compat_str):
3026                     src_type = src_type.lower()
3027                 ext = determine_ext(src).lower()
3028                 if src_type == 'video/youtube':
3029                     return self.url_result(src, YoutubeIE.ie_key())
3030                 if src_type == 'application/dash+xml' or ext == 'mpd':
3031                     formats.extend(self._extract_mpd_formats(
3032                         src, video_id, mpd_id='dash', fatal=False))
3033                 elif src_type == 'application/x-mpegurl' or ext == 'm3u8':
3034                     formats.extend(self._extract_m3u8_formats(
3035                         src, video_id, 'mp4', entry_protocol='m3u8_native',
3036                         m3u8_id='hls', fatal=False))
3037                 else:
3038                     formats.append({
3039                         'url': src,
3040                         'ext': (mimetype2ext(src_type) or
3041                                 ext if ext in KNOWN_EXTENSIONS else 'mp4'),
3042                     })
3043             if formats:
3044                 self._sort_formats(formats)
3045                 info_dict['formats'] = formats
3046                 return info_dict
3047
3048         # Looking for http://schema.org/VideoObject
3049         json_ld = self._search_json_ld(
3050             webpage, video_id, default={}, expected_type='VideoObject')
3051         if json_ld.get('url'):
3052             return merge_dicts(json_ld, info_dict)
3053
3054         def check_video(vurl):
3055             if YoutubeIE.suitable(vurl):
3056                 return True
3057             if RtmpIE.suitable(vurl):
3058                 return True
3059             vpath = compat_urlparse.urlparse(vurl).path
3060             vext = determine_ext(vpath)
3061             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js', 'xml')
3062
3063         def filter_video(urls):
3064             return list(filter(check_video, urls))
3065
3066         # Start with something easy: JW Player in SWFObject
3067         found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
3068         if not found:
3069             # Look for gorilla-vid style embedding
3070             found = filter_video(re.findall(r'''(?sx)
3071                 (?:
3072                     jw_plugins|
3073                     JWPlayerOptions|
3074                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
3075                 )
3076                 .*?
3077                 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
3078         if not found:
3079             # Broaden the search a little bit
3080             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
3081         if not found:
3082             # Broaden the findall a little bit: JWPlayer JS loader
3083             found = filter_video(re.findall(
3084                 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
3085         if not found:
3086             # Flow player
3087             found = filter_video(re.findall(r'''(?xs)
3088                 flowplayer\("[^"]+",\s*
3089                     \{[^}]+?\}\s*,
3090                     \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
3091                         ["']?url["']?\s*:\s*["']([^"']+)["']
3092             ''', webpage))
3093         if not found:
3094             # Cinerama player
3095             found = re.findall(
3096                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
3097         if not found:
3098             # Try to find twitter cards info
3099             # twitter:player:stream should be checked before twitter:player since
3100             # it is expected to contain a raw stream (see
3101             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
3102             found = filter_video(re.findall(
3103                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
3104         if not found:
3105             # We look for Open Graph info:
3106             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
3107             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
3108             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
3109             if m_video_type is not None:
3110                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
3111         if not found:
3112             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
3113             found = re.search(
3114                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
3115                 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
3116                 webpage)
3117             if not found:
3118                 # Look also in Refresh HTTP header
3119                 refresh_header = head_response.headers.get('Refresh')
3120                 if refresh_header:
3121                     # In python 2 response HTTP headers are bytestrings
3122                     if sys.version_info < (3, 0) and isinstance(refresh_header, str):
3123                         refresh_header = refresh_header.decode('iso-8859-1')
3124                     found = re.search(REDIRECT_REGEX, refresh_header)
3125             if found:
3126                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
3127                 if new_url != url:
3128                     self.report_following_redirect(new_url)
3129                     return {
3130                         '_type': 'url',
3131                         'url': new_url,
3132                     }
3133                 else:
3134                     found = None
3135
3136         if not found:
3137             # twitter:player is a https URL to iframe player that may or may not
3138             # be supported by youtube-dl thus this is checked the very last (see
3139             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
3140             embed_url = self._html_search_meta('twitter:player', webpage, default=None)
3141             if embed_url and embed_url != url:
3142                 return self.url_result(embed_url)
3143
3144         if not found:
3145             raise UnsupportedError(url)
3146
3147         entries = []
3148         for video_url in orderedSet(found):
3149             video_url = unescapeHTML(video_url)
3150             video_url = video_url.replace('\\/', '/')
3151             video_url = compat_urlparse.urljoin(url, video_url)
3152             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
3153
3154             # Sometimes, jwplayer extraction will result in a YouTube URL
3155             if YoutubeIE.suitable(video_url):
3156                 entries.append(self.url_result(video_url, 'Youtube'))
3157                 continue
3158
3159             # here's a fun little line of code for you:
3160             video_id = os.path.splitext(video_id)[0]
3161
3162             entry_info_dict = {
3163                 'id': video_id,
3164                 'uploader': video_uploader,
3165                 'title': video_title,
3166                 'age_limit': age_limit,
3167             }
3168
3169             if RtmpIE.suitable(video_url):
3170                 entry_info_dict.update({
3171                     '_type': 'url_transparent',
3172                     'ie_key': RtmpIE.ie_key(),
3173                     'url': video_url,
3174                 })
3175                 entries.append(entry_info_dict)
3176                 continue
3177
3178             ext = determine_ext(video_url)
3179             if ext == 'smil':
3180                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
3181             elif ext == 'xspf':
3182                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
3183             elif ext == 'm3u8':
3184                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
3185             elif ext == 'mpd':
3186                 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
3187             elif ext == 'f4m':
3188                 entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
3189             elif re.search(r'(?i)\.(?:ism|smil)/manifest', video_url) and video_url != url:
3190                 # Just matching .ism/manifest is not enough to be reliably sure
3191                 # whether it's actually an ISM manifest or some other streaming
3192                 # manifest since there are various streaming URL formats
3193                 # possible (see [1]) as well as some other shenanigans like
3194                 # .smil/manifest URLs that actually serve an ISM (see [2]) and
3195                 # so on.
3196                 # Thus the most reasonable way to solve this is to delegate
3197                 # to generic extractor in order to look into the contents of
3198                 # the manifest itself.
3199                 # 1. https://azure.microsoft.com/en-us/documentation/articles/media-services-deliver-content-overview/#streaming-url-formats
3200                 # 2. https://svs.itworkscdn.net/lbcivod/smil:itwfcdn/lbci/170976.smil/Manifest
3201                 entry_info_dict = self.url_result(
3202                     smuggle_url(video_url, {'to_generic': True}),
3203                     GenericIE.ie_key())
3204             else:
3205                 entry_info_dict['url'] = video_url
3206
3207             if entry_info_dict.get('formats'):
3208                 self._sort_formats(entry_info_dict['formats'])
3209
3210             entries.append(entry_info_dict)
3211
3212         if len(entries) == 1:
3213             return entries[0]
3214         else:
3215             for num, e in enumerate(entries, start=1):
3216                 # 'url' results don't have a title
3217                 if e.get('title') is not None:
3218                     e['title'] = '%s (%d)' % (e['title'], num)
3219             return {
3220                 '_type': 'playlist',
3221                 'entries': entries,
3222             }