Move duplicate check to generic.py

[youtube-dl] / youtube_dl / extractor / generic.py

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d7f0a4d39697f33063f23278d6cd8a80cf6e3368..dfa8d6153303ecf250c0b41fe4fe4640ec4e7d65 100644 (file)
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -260,6 +260,20 @@ class GenericIE(InfoExtractor):
                 'uploader': 'Spi0n',
             },
             'add_ie': ['Dailymotion'],
+        },
+        # YouTube embed via <data-embed-url="">
+        {
+            'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
+            'md5': 'c267b1ab6d736057d64babaa37e07a66',
+            'info_dict': {
+                'id': 'Ybd-qmqYYpA',
+                'ext': 'mp4',
+                'title': 'Asphalt 8: Airborne -  Chinese Great Wall - Android Game Trailer',
+                'uploader': 'gameloftandroid',
+                'uploader_id': 'gameloftandroid',
+                'upload_date': '20140321',
+                'description': 'md5:9c6dca5dd75b7131ce482ccf080749d6'
+            }
         }
     ]
 
@@ -348,10 +362,7 @@ class GenericIE(InfoExtractor):
         if url.startswith('//'):
             return {
                 '_type': 'url',
-                'url': (
-                    'http:'
-                    if self._downloader.params.get('prefer_insecure', False)
-                    else 'https:') + url,
+                'url': self.http_scheme() + url,
             }
 
         parsed_url = compat_urlparse.urlparse(url)
@@ -476,13 +487,21 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)
-            (?:<iframe[^>]+?src=|embedSWF\(\s*)
+            (?:<iframe[^>]+?src=|data-video-url=|embedSWF\(\s*)
             (["\'])(?P<url>(?:https?:)?//(?:www\.)?youtube\.com/
                 (?:embed|v)/.+?)
             \1''', webpage)
         if matches:
             urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
                      for tuppl in matches]
+            # First, ensure we have a duplicate free list of entries
+            seen = set()
+            new_list = []
+            theurl = tuple(url.items())
+            if theurl not in seen:
+                seen.add(theurl)
+                new_list.append(url)
+                urlrs = new_list
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
@@ -492,6 +511,14 @@ class GenericIE(InfoExtractor):
         if matches:
             urlrs = [self.url_result(unescapeHTML(tuppl[1]))
                      for tuppl in matches]
+            # First, ensure we have a duplicate free list of entries
+            seen = set()
+            new_list = []
+            theurl = tuple(url.items())
+            if theurl not in seen:
+                seen.add(theurl)
+                new_list.append(url)
+                urlrs = new_list
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
@@ -563,7 +590,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded NovaMov-based player
         mobj = re.search(
-            r'''(?x)<iframe[^>]+?src=(["\'])
+            r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
                     (?P<url>http://(?:(?:embed|www)\.)?
                         (?:novamov\.com|
                            nowvideo\.(?:ch|sx|eu|at|ag|co)|
@@ -604,6 +631,14 @@ class GenericIE(InfoExtractor):
         if matches:
             urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie')
                      for eurl in matches]
+            # First, ensure we have a duplicate free list of entries
+            seen = set()
+            new_list = []
+            theurl = tuple(url.items())
+            if theurl not in seen:
+                seen.add(theurl)
+                new_list.append(url)
+                urlrs = new_list
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
@@ -636,6 +671,14 @@ class GenericIE(InfoExtractor):
         if smotri_url:
             return self.url_result(smotri_url, 'Smotri')
 
+        # Look for embeded soundcloud player
+        mobj = re.search(
+            r'<iframe src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
+            webpage)
+        if mobj is not None:
+            url = unescapeHTML(mobj.group('url'))
+            return self.url_result(url)
+
         # Start with something easy: JW Player in SWFObject
         found = re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
         if not found:
@@ -667,7 +710,7 @@ class GenericIE(InfoExtractor):
             # HTML5 video
             found = re.findall(r'(?s)<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage)
         if not found:
-            found = re.findall(
+            found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
                 r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'([^\']+)\'"',
                 webpage)