Merge remote-tracking branch 'anovicecodemonkey/generic-data-video-url'
authorPhilipp Hagemeister <phihag@phihag.de>
Fri, 22 Aug 2014 15:40:36 +0000 (17:40 +0200)
committerPhilipp Hagemeister <phihag@phihag.de>
Fri, 22 Aug 2014 15:40:36 +0000 (17:40 +0200)
Conflicts:
youtube_dl/extractor/generic.py

1  2 
youtube_dl/extractor/generic.py

index 8e915735eaedcacaff85b4408fb93deb79bac16b,dfa8d6153303ecf250c0b41fe4fe4640ec4e7d65..7246ea7fc288754a529b4536e9bc8c2c055ff940
@@@ -261,34 -261,20 +261,48 @@@ class GenericIE(InfoExtractor)
              },
              'add_ie': ['Dailymotion'],
          },
 +        # YouTube embed
 +        {
 +            'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
 +            'info_dict': {
 +                'id': 'FXRb4ykk4S0',
 +                'ext': 'mp4',
 +                'title': 'The NBL Auction 2014',
 +                'uploader': 'BADMINTON England',
 +                'uploader_id': 'BADMINTONEvents',
 +                'upload_date': '20140603',
 +                'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
 +            },
 +            'add_ie': ['Youtube'],
 +            'params': {
 +                'skip_download': True,
 +            }
 +        },
 +        # MTVSercices embed
 +        {
 +            'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
 +            'md5': '35727f82f58c76d996fc188f9755b0d5',
 +            'info_dict': {
 +                'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
 +                'ext': 'mp4',
 +                'title': 'Review',
 +                'description': 'Mario\'s life in the fast lane has never looked so good.',
 +            },
 +        },
+         # YouTube embed via <data-embed-url="">
+         {
+             'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
+             'md5': 'c267b1ab6d736057d64babaa37e07a66',
+             'info_dict': {
+                 'id': 'Ybd-qmqYYpA',
+                 'ext': 'mp4',
+                 'title': 'Asphalt 8: Airborne -  Chinese Great Wall - Android Game Trailer',
+                 'uploader': 'gameloftandroid',
+                 'uploader_id': 'gameloftandroid',
+                 'upload_date': '20140321',
+                 'description': 'md5:9c6dca5dd75b7131ce482ccf080749d6'
+             }
+         }
      ]
  
      def report_download_webpage(self, video_id):
          if not parsed_url.scheme:
              default_search = self._downloader.params.get('default_search')
              if default_search is None:
 -                default_search = 'auto_warning'
 +                default_search = 'fixup_error'
  
 -            if default_search in ('auto', 'auto_warning'):
 +            if default_search in ('auto', 'auto_warning', 'fixup_error'):
                  if '/' in url:
                      self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
                      return self.url_result('http://' + url)
 -                else:
 +                elif default_search != 'fixup_error':
                      if default_search == 'auto_warning':
 -                        self._downloader.report_warning(
 -                            'Falling back to youtube search for  %s . Set --default-search to "auto" to suppress this warning.' % url)
 +                        if re.match(r'^(?:url|URL)$', url):
 +                            raise ExtractorError(
 +                                'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
 +                                expected=True)
 +                        else:
 +                            self._downloader.report_warning(
 +                                'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
                      return self.url_result('ytsearch:' + url)
 +
 +            if default_search in ('error', 'fixup_error'):
 +                raise ExtractorError(
 +                    ('%r is not a valid URL. '
 +                     'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
 +                    ) % (url, url), expected=True)
              else:
                  assert ':' in default_search
                  return self.url_result(default_search + url)
  
          # Look for embedded YouTube player
          matches = re.findall(r'''(?x)
 -            (?:<iframe[^>]+?src=|data-video-url=|embedSWF\(\s*)
 -            (["\'])(?P<url>(?:https?:)?//(?:www\.)?youtube\.com/
 +            (?:
 +                <iframe[^>]+?src=|
++                data-video-url=|
 +                <embed[^>]+?src=|
 +                embedSWF\(?:\s*
 +            )
 +            (["\'])
 +                (?P<url>(?:https?:)?//(?:www\.)?youtube\.com/
                  (?:embed|v)/.+?)
              \1''', webpage)
          if matches:
              urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
                       for tuppl in matches]
+             # First, ensure we have a duplicate free list of entries
+             seen = set()
+             new_list = []
+             theurl = tuple(url.items())
+             if theurl not in seen:
+                 seen.add(theurl)
+                 new_list.append(url)
+                 urlrs = new_list
              return self.playlist_result(
                  urlrs, playlist_id=video_id, playlist_title=video_title)
  
          if matches:
              urlrs = [self.url_result(unescapeHTML(tuppl[1]))
                       for tuppl in matches]
+             # First, ensure we have a duplicate free list of entries
+             seen = set()
+             new_list = []
+             theurl = tuple(url.items())
+             if theurl not in seen:
+                 seen.add(theurl)
+                 new_list.append(url)
+                 urlrs = new_list
              return self.playlist_result(
                  urlrs, playlist_id=video_id, playlist_title=video_title)
  
          if mobj is not None:
              return self.url_result(mobj.group('url'), 'VK')
  
 +        # Look for embedded ivi player
 +        mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
 +        if mobj is not None:
 +            return self.url_result(mobj.group('url'), 'Ivi')
 +
          # Look for embedded Huffington Post player
          mobj = re.search(
              r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
          if matches:
              urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie')
                       for eurl in matches]
+             # First, ensure we have a duplicate free list of entries
+             seen = set()
+             new_list = []
+             theurl = tuple(url.items())
+             if theurl not in seen:
+                 seen.add(theurl)
+                 new_list.append(url)
+                 urlrs = new_list
              return self.playlist_result(
                  urlrs, playlist_id=video_id, playlist_title=video_title)
  
              url = unescapeHTML(mobj.group('url'))
              return self.url_result(url)
  
 +        # Look for embedded vulture.com player
 +        mobj = re.search(
 +            r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
 +            webpage)
 +        if mobj is not None:
 +            url = unescapeHTML(mobj.group('url'))
 +            return self.url_result(url, ie='Vulture')
 +
 +        # Look for embedded mtvservices player
 +        mobj = re.search(
 +            r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
 +            webpage)
 +        if mobj is not None:
 +            url = unescapeHTML(mobj.group('url'))
 +            return self.url_result(url, ie='MTVServicesEmbedded')
 +
 +        # Look for embedded yahoo player
 +        mobj = re.search(
 +            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
 +            webpage)
 +        if mobj is not None:
 +            return self.url_result(mobj.group('url'), 'Yahoo')
 +
          # Start with something easy: JW Player in SWFObject
          found = re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
          if not found: