From: Sergey M․ <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 19:19:39 +0000 (+0600)
Subject: [extractor/common] Add _remove_duplicate_formats
X-Git-Url: http://git.bitcoin.ninja/?a=commitdiff_plain;h=f5bdb4444338f1462c2bc7f79b4ad7e260fe56a8;p=youtube-dl

[extractor/common] Add _remove_duplicate_formats
---

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c85dcda0c..14f575635 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -900,6 +900,16 @@ class InfoExtractor(object):
                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
                 formats)
 
+    @staticmethod
+    def _remove_duplicate_formats(formats):
+        format_urls = set()
+        unique_formats = []
+        for f in formats:
+            if f['url'] not in format_urls:
+                format_urls.add(f['url'])
+                unique_formats.append(f)
+        formats[:] = unique_formats
+
     def _is_valid_url(self, url, video_id, item='video'):
         url = self._proto_relative_url(url, scheme='http:')
         # For now assume non HTTP(S) URLs always valid