Release 2012.01.08
authorPhilipp Hagemeister <phihag@phihag.de>
Sun, 8 Jan 2012 16:20:55 +0000 (17:20 +0100)
committerPhilipp Hagemeister <phihag@phihag.de>
Sun, 8 Jan 2012 16:20:55 +0000 (17:20 +0100)
LATEST_VERSION
youtube-dl
youtube_dl/__init__.py

index 0c82f59b5202254cf7bd07f56d8bb7991054f25d..a4c711a25ff7d1cde5a0c52cd381a5fe66466211 100644 (file)
@@ -1 +1 @@
-2012.01.05
+2012.01.08
index 818a3a73a20ec18a7e0aa55cf687a765e09eb0df..b22f1cac274fa165563f1f7e32b5a5ee474644f6 100755 (executable)
@@ -18,7 +18,7 @@ __authors__  = (
        )
 
 __license__ = 'Public Domain'
-__version__ = '2012.01.05'
+__version__ = '2012.01.08'
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 
@@ -259,14 +259,14 @@ def sanitize_open(filename, open_mode):
                                import msvcrt
                                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
                        return (sys.stdout, filename)
-               stream = open(filename, open_mode)
+               stream = open(_encodeFilename(filename), open_mode)
                return (stream, filename)
        except (IOError, OSError), err:
                # In case of error, try to remove win32 forbidden chars
                filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
 
                # An exception here should be caught in the caller
-               stream = open(filename, open_mode)
+               stream = open(_encodeFilename(filename), open_mode)
                return (stream, filename)
 
 
@@ -291,13 +291,21 @@ def _orderedSet(iterable):
        return res
 
 def _unescapeHTML(s):
-    """
-    @param s a string (of type unicode)
-    """
-    assert type(s) == type(u'')
+       """
+       @param s a string (of type unicode)
+       """
+       assert type(s) == type(u'')
+
+       htmlParser = HTMLParser.HTMLParser()
+       return htmlParser.unescape(s)
 
-    htmlParser = HTMLParser.HTMLParser()
-    return htmlParser.unescape(s)
+def _encodeFilename(s):
+       """
+       @param s The name of the file (of type unicode)
+       """
+
+       assert type(s) == type(u'')
+       return s.encode(sys.getfilesystemencoding(), 'ignore')
 
 class DownloadError(Exception):
        """Download Error exception.
@@ -563,16 +571,17 @@ class FileDownloader(object):
                self._pps.append(pp)
                pp.set_downloader(self)
 
-       def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
+       def to_screen(self, message, skip_eol=False):
                """Print message to stdout if not in quiet mode."""
-               try:
-                       if not self.params.get('quiet', False):
-                               terminator = [u'\n', u''][skip_eol]
-                               print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
+               assert type(message) == type(u'')
+               if not self.params.get('quiet', False):
+                       terminator = [u'\n', u''][skip_eol]
+                       output = message + terminator
+
+                       if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+                               output = output.encode(preferredencoding(), 'ignore')
+                       self._screen_file.write(output)
                        self._screen_file.flush()
-               except (UnicodeEncodeError), err:
-                       if not ignore_encoding_errors:
-                               raise
 
        def to_stderr(self, message):
                """Print message to stderr."""
@@ -622,7 +631,7 @@ class FileDownloader(object):
        def temp_name(self, filename):
                """Returns a temporary filename for the given filename."""
                if self.params.get('nopart', False) or filename == u'-' or \
-                               (os.path.exists(filename) and not os.path.isfile(filename)):
+                               (os.path.exists(_encodeFilename(filename)) and not os.path.isfile(_encodeFilename(filename))):
                        return filename
                return filename + u'.part'
 
@@ -635,7 +644,7 @@ class FileDownloader(object):
                try:
                        if old_filename == new_filename:
                                return
-                       os.rename(old_filename, new_filename)
+                       os.rename(_encodeFilename(old_filename), _encodeFilename(new_filename))
                except (IOError, OSError), err:
                        self.trouble(u'ERROR: unable to rename file')
 
@@ -643,7 +652,7 @@ class FileDownloader(object):
                """Try to set the last-modified time of the given file."""
                if last_modified_hdr is None:
                        return
-               if not os.path.isfile(filename):
+               if not os.path.isfile(_encodeFilename(filename)):
                        return
                timestr = last_modified_hdr
                if timestr is None:
@@ -659,15 +668,15 @@ class FileDownloader(object):
 
        def report_writedescription(self, descfn):
                """ Report that the description file is being written """
-               self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True)
+               self.to_screen(u'[info] Writing video description to: ' + descfn)
 
        def report_writeinfojson(self, infofn):
                """ Report that the metadata file has been written """
-               self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True)
+               self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 
        def report_destination(self, filename):
                """Report destination filename."""
-               self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
+               self.to_screen(u'[download] Destination: ' + filename)
 
        def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
                """Report download progress."""
@@ -769,8 +778,8 @@ class FileDownloader(object):
                        return
 
                try:
-                       dn = os.path.dirname(filename)
-                       if dn != '' and not os.path.exists(dn):
+                       dn = os.path.dirname(_encodeFilename(filename))
+                       if dn != '' and not os.path.exists(dn): # dn is already encoded
                                os.makedirs(dn)
                except (OSError, IOError), err:
                        self.trouble(u'ERROR: unable to create directory ' + unicode(err))
@@ -778,9 +787,9 @@ class FileDownloader(object):
 
                if self.params.get('writedescription', False):
                        try:
-                               descfn = filename + '.description'
+                               descfn = filename + u'.description'
                                self.report_writedescription(descfn)
-                               descfile = open(descfn, 'wb')
+                               descfile = open(_encodeFilename(descfn), 'wb')
                                try:
                                        descfile.write(info_dict['description'].encode('utf-8'))
                                finally:
@@ -790,7 +799,7 @@ class FileDownloader(object):
                                return
 
                if self.params.get('writeinfojson', False):
-                       infofn = filename + '.info.json'
+                       infofn = filename + u'.info.json'
                        self.report_writeinfojson(infofn)
                        try:
                                json.dump
@@ -798,7 +807,7 @@ class FileDownloader(object):
                                self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
                                return
                        try:
-                               infof = open(infofn, 'wb')
+                               infof = open(_encodeFilename(infofn), 'wb')
                                try:
                                        json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
                                        json.dump(json_info_dict, infof)
@@ -809,7 +818,7 @@ class FileDownloader(object):
                                return
 
                if not self.params.get('skip_download', False):
-                       if self.params.get('nooverwrites', False) and os.path.exists(filename):
+                       if self.params.get('nooverwrites', False) and os.path.exists(_encodeFilename(filename)):
                                success = True
                        else:
                                try:
@@ -882,11 +891,11 @@ class FileDownloader(object):
                basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
                retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
                while retval == 2 or retval == 1:
-                       prevsize = os.path.getsize(tmpfilename)
+                       prevsize = os.path.getsize(_encodeFilename(tmpfilename))
                        self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
                        time.sleep(5.0) # This seems to be needed
                        retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
-                       cursize = os.path.getsize(tmpfilename)
+                       cursize = os.path.getsize(_encodeFilename(tmpfilename))
                        if prevsize == cursize and retval == 1:
                                break
                         # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
@@ -895,7 +904,7 @@ class FileDownloader(object):
                                retval = 0
                                break
                if retval == 0:
-                       self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
+                       self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(_encodeFilename(tmpfilename)))
                        self.try_rename(tmpfilename, filename)
                        return True
                else:
@@ -907,7 +916,7 @@ class FileDownloader(object):
                player_url = info_dict.get('player_url', None)
 
                # Check file already present
-               if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
+               if self.params.get('continuedl', False) and os.path.isfile(_encodeFilename(filename)) and not self.params.get('nopart', False):
                        self.report_file_already_downloaded(filename)
                        return True
 
@@ -924,8 +933,8 @@ class FileDownloader(object):
                request = urllib2.Request(url, None, headers)
 
                # Establish possible resume length
-               if os.path.isfile(tmpfilename):
-                       resume_len = os.path.getsize(tmpfilename)
+               if os.path.isfile(_encodeFilename(tmpfilename)):
+                       resume_len = os.path.getsize(_encodeFilename(tmpfilename))
                else:
                        resume_len = 0
 
@@ -3750,17 +3759,17 @@ class MixcloudIE(InfoExtractor):
                try:
                        # Process file information
                        self._downloader.process_info({
-                               'id':           file_id.decode('utf-8'),
-                               'url':          file_url.decode('utf-8'),
+                               'id': file_id.decode('utf-8'),
+                               'url': file_url.decode('utf-8'),
                                'uploader':     uploader.decode('utf-8'),
-                               'upload_date':  u'NA',
-                               'title':        json_data['name'],
-                               'stitle':       _simplify_title(json_data['name']),
-                               'ext':          file_url.split('.')[-1].decode('utf-8'),
-                               'format':       (format_param is None and u'NA' or format_param.decode('utf-8')),
-                               'thumbnail':    json_data['thumbnail_url'],
-                               'description':  json_data['description'],
-                               'player_url':   player_url.decode('utf-8'),
+                               'upload_date': u'NA',
+                               'title': json_data['name'],
+                               'stitle': _simplify_title(json_data['name']),
+                               'ext': file_url.split('.')[-1].decode('utf-8'),
+                               'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
+                               'thumbnail': json_data['thumbnail_url'],
+                               'description': json_data['description'],
+                               'player_url': player_url.decode('utf-8'),
                        })
                except UnavailableVideoError, err:
                        self._downloader.trouble(u'ERROR: unable to download file')
@@ -3884,6 +3893,100 @@ class StanfordOpenClassroomIE(InfoExtractor):
                                assert entry['type'] == 'reference'
                                self.extract(entry['url'])
 
+class MTVIE(InfoExtractor):
+       """Information extractor for MTV.com"""
+
+       _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
+       IE_NAME = u'mtv'
+
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _real_extract(self, url):
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               if not mobj.group('proto'):
+                       url = 'http://' + url
+               video_id = mobj.group('videoid')
+               self.report_webpage(video_id)
+
+               request = urllib2.Request(url)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+
+               mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract song name')
+                       return
+               song_name = _unescapeHTML(mobj.group(1).decode('iso-8859-1'))
+               mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract performer')
+                       return
+               performer = _unescapeHTML(mobj.group(1).decode('iso-8859-1'))
+               video_title = performer + ' - ' + song_name 
+
+               mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to mtvn_uri')
+                       return
+               mtvn_uri = mobj.group(1)
+
+               mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract content id')
+                       return
+               content_id = mobj.group(1)
+
+               videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
+               self.report_extraction(video_id)
+               request = urllib2.Request(videogen_url)
+               try:
+                       metadataXml = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err))
+                       return
+
+               mdoc = xml.etree.ElementTree.fromstring(metadataXml)
+               renditions = mdoc.findall('.//rendition')
+
+               # For now, always pick the highest quality.
+               rendition = renditions[-1]
+
+               try:
+                       _,_,ext = rendition.attrib['type'].partition('/')
+                       format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
+                       video_url = rendition.find('./src').text
+               except KeyError:
+                       self._downloader.trouble('Invalid rendition field.')
+                       return
+
+               self._downloader.increment_downloads()
+               info = {
+                       'id': video_id,
+                       'url': video_url,
+                       'uploader': performer,
+                       'title': video_title,
+                       'stitle': _simplify_title(video_title),
+                       'ext': ext,
+                       'format': format,
+               }
+
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+
 
 class PostProcessor(object):
        """Post Processor class.
@@ -3948,7 +4051,7 @@ class FFmpegExtractAudioPP(PostProcessor):
        @staticmethod
        def get_audio_codec(path):
                try:
-                       cmd = ['ffprobe', '-show_streams', '--', path]
+                       cmd = ['ffprobe', '-show_streams', '--', _encodeFilename(path)]
                        handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
                        output = handle.communicate()[0]
                        if handle.wait() != 0:
@@ -3969,7 +4072,7 @@ class FFmpegExtractAudioPP(PostProcessor):
                        acodec_opts = []
                else:
                        acodec_opts = ['-acodec', codec]
-               cmd = ['ffmpeg', '-y', '-i', path, '-vn'] + acodec_opts + more_opts + ['--', out_path]
+               cmd = ['ffmpeg', '-y', '-i', _encodeFilename(path), '-vn'] + acodec_opts + more_opts + ['--', _encodeFilename(out_path)]
                try:
                        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                        stdout,stderr = p.communicate()
@@ -4030,9 +4133,9 @@ class FFmpegExtractAudioPP(PostProcessor):
                                extension = 'wav'
                                more_opts += ['-f', 'wav']
 
-               (prefix, ext) = os.path.splitext(path)
-               new_path = prefix + '.' + extension
-               self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path)
+               prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+               new_path = prefix + sep + extension
+               self._downloader.to_screen(u'[ffmpeg] Destination: ' + new_path)
                try:
                        self.run_ffmpeg(path, new_path, acodec, more_opts)
                except:
@@ -4046,13 +4149,13 @@ class FFmpegExtractAudioPP(PostProcessor):
                # Try to update the date time for extracted audio file.
                if information.get('filetime') is not None:
                        try:
-                               os.utime(new_path, (time.time(), information['filetime']))
+                               os.utime(_encodeFilename(new_path), (time.time(), information['filetime']))
                        except:
                                self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
 
                if not self._keepvideo:
                        try:
-                               os.remove(path)
+                               os.remove(_encodeFilename(path))
                        except (IOError, OSError):
                                self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
                                return None
@@ -4100,9 +4203,9 @@ def parseOpts():
        import optparse
        import shlex
 
-       def _readOptions(filename):
+       def _readOptions(filename_bytes):
                try:
-                       optionf = open(filename)
+                       optionf = open(filename_bytes)
                except IOError:
                        return [] # silently skip if file is not present
                try:
@@ -4336,6 +4439,7 @@ def gen_extractors():
                InfoQIE(),
                MixcloudIE(),
                StanfordOpenClassroomIE(),
+               MTVIE(),
 
                GenericIE()
        ]
index 0f9eeccbf017f9d3edb8ce91f470e4ec56f05120..b22f1cac274fa165563f1f7e32b5a5ee474644f6 100755 (executable)
@@ -18,7 +18,7 @@ __authors__  = (
        )
 
 __license__ = 'Public Domain'
-__version__ = '2012.01.05'
+__version__ = '2012.01.08'
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'