- def prepare_filename(self, info_dict):
- """Generate the output filename."""
- try:
- template_dict = dict(info_dict)
-
- template_dict['epoch'] = int(time.time())
- autonumber_size = self.params.get('autonumber_size')
- if autonumber_size is None:
- autonumber_size = 5
- autonumber_templ = u'%0' + str(autonumber_size) + u'd'
- template_dict['autonumber'] = autonumber_templ % self._num_downloads
- if template_dict['playlist_index'] is not None:
- template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
-
- sanitize = lambda k,v: sanitize_filename(
- u'NA' if v is None else compat_str(v),
- restricted=self.params.get('restrictfilenames'),
- is_id=(k==u'id'))
- template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
-
- filename = self.params['outtmpl'] % template_dict
- return filename
- except KeyError as err:
- self.trouble(u'ERROR: Erroneous output template')
- return None
- except ValueError as err:
- self.trouble(u'ERROR: Insufficient system charset ' + repr(preferredencoding()))
- return None
-
- def _match_entry(self, info_dict):
- """ Returns None iff the file should be downloaded """
-
- title = info_dict['title']
- matchtitle = self.params.get('matchtitle', False)
- if matchtitle:
- if not re.search(matchtitle, title, re.IGNORECASE):
- return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
- rejecttitle = self.params.get('rejecttitle', False)
- if rejecttitle:
- if re.search(rejecttitle, title, re.IGNORECASE):
- return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
- return None
-
- def extract_info(self, url, download = True, ie_name = None):
- '''
- Returns a list with a dictionary for each video we find.
- If 'download', also downloads the videos.
- '''
- suitable_found = False
-
- #We copy the original list
- ies = list(self._ies)
-
- if ie_name is not None:
- #We put in the first place the given info extractor
- first_ie = get_info_extractor(ie_name)()
- first_ie.set_downloader(self)
- ies.insert(0, first_ie)
-
- for ie in ies:
- # Go to next InfoExtractor if not suitable
- if not ie.suitable(url):
- continue
-
- # Warn if the _WORKING attribute is False
- if not ie.working():
- self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
- u'and will probably not work. If you want to go on, use the -i option.')
-
- # Suitable InfoExtractor found
- suitable_found = True
-
- # Extract information from URL and process it
- try:
- ie_results = ie.extract(url)
- results = []
- for ie_result in ie_results:
- if not 'extractor' in ie_result:
- #The extractor has already been set somewhere else
- ie_result['extractor'] = ie.IE_NAME
- results.append(self.process_ie_result(ie_result, download))
- return results
- except ExtractorError as de: # An error we somewhat expected
- self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
- break
- except Exception as e:
- if self.params.get('ignoreerrors', False):
- self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
- break
- else:
- raise
- if not suitable_found:
- self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
-
- def process_ie_result(self, ie_result, download = True):
- """
- Take the result of the ie and return a list of videos.
- For url elements it will search the suitable ie and get the videos
- For playlist elements it will process each of the elements of the 'entries' key
-
- It will also download the videos if 'download'.
- """
- result_type = ie_result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system
- if result_type == 'video':
- if 'playlist' not in ie_result:
- #It isn't part of a playlist
- ie_result['playlist'] = None
- ie_result['playlist_index'] = None
- if download:
- #Do the download:
- self.process_info(ie_result)
- return ie_result
- elif result_type == 'url':
- #We get the video pointed by the url
- result = self.extract_info(ie_result['url'], download, ie_name = ie_result['ie_key'])[0]
- return result
- elif result_type == 'playlist':
- #We process each entry in the playlist
- playlist = ie_result.get('title', None) or ie_result.get('id', None)
- self.to_screen(u'[download] Downloading playlist: %s' % playlist)
-
- playlist_results = []
-
- n_all_entries = len(ie_result['entries'])
- playliststart = self.params.get('playliststart', 1) - 1
- playlistend = self.params.get('playlistend', -1)
-
- if playlistend == -1:
- entries = ie_result['entries'][playliststart:]
- else:
- entries = ie_result['entries'][playliststart:playlistend]
-
- n_entries = len(entries)
-
- self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
- (ie_result['extractor'], playlist, n_all_entries, n_entries))
-
- for i,entry in enumerate(entries,1):
- self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
- entry_result = self.process_ie_result(entry, False)
- entry_result['playlist'] = playlist
- entry_result['playlist_index'] = i + playliststart
- #We must do the download here to correctly set the 'playlist' key
- if download:
- self.process_info(entry_result)
- playlist_results.append(entry_result)
- result = ie_result.copy()
- result['entries'] = playlist_results
- return result
-
- def process_info(self, info_dict):
- """Process a single dictionary returned by an InfoExtractor."""
-
- #We increment the download the download count here to match the previous behaviour.
- self.increment_downloads()
-
- info_dict['fulltitle'] = info_dict['title']
- if len(info_dict['title']) > 200:
- info_dict['title'] = info_dict['title'][:197] + u'...'
-
- # Keep for backwards compatibility
- info_dict['stitle'] = info_dict['title']
-
- if not 'format' in info_dict:
- info_dict['format'] = info_dict['ext']
-
- reason = self._match_entry(info_dict)
- if reason is not None:
- self.to_screen(u'[download] ' + reason)
- return
-
- max_downloads = self.params.get('max_downloads')
- if max_downloads is not None:
- if self._num_downloads > int(max_downloads):
- raise MaxDownloadsReached()
-
- filename = self.prepare_filename(info_dict)
-
- # Forced printings
- if self.params.get('forcetitle', False):
- compat_print(info_dict['title'])
- if self.params.get('forceurl', False):
- compat_print(info_dict['url'])
- if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
- compat_print(info_dict['thumbnail'])
- if self.params.get('forcedescription', False) and 'description' in info_dict:
- compat_print(info_dict['description'])
- if self.params.get('forcefilename', False) and filename is not None:
- compat_print(filename)
- if self.params.get('forceformat', False):
- compat_print(info_dict['format'])
-
- # Do nothing else if in simulate mode
- if self.params.get('simulate', False):
- return
-
- if filename is None:
- return
-
- try:
- dn = os.path.dirname(encodeFilename(filename))
- if dn != '' and not os.path.exists(dn): # dn is already encoded
- os.makedirs(dn)
- except (OSError, IOError) as err:
- self.report_error(u'unable to create directory ' + compat_str(err))
- return
-
- if self.params.get('writedescription', False):
- try:
- descfn = filename + u'.description'
- self.report_writedescription(descfn)
- with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
- descfile.write(info_dict['description'])
- except (OSError, IOError):
- self.report_error(u'Cannot write description file ' + descfn)
- return
-
- if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
- # subtitles download errors are already managed as troubles in relevant IE
- # that way it will silently go on when used with unsupporting IE
- subtitle = info_dict['subtitles'][0]
- (sub_error, sub_lang, sub) = subtitle
- sub_format = self.params.get('subtitlesformat')
- if sub_error:
- self.report_warning("Some error while getting the subtitles")
- else:
- try:
- sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
- self.report_writesubtitles(sub_filename)
- with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
- subfile.write(sub)
- except (OSError, IOError):
- self.report_error(u'Cannot write subtitles file ' + descfn)
- return
- if self.params.get('onlysubtitles', False):
- return
-
- if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
- subtitles = info_dict['subtitles']
- sub_format = self.params.get('subtitlesformat')
- for subtitle in subtitles:
- (sub_error, sub_lang, sub) = subtitle
- if sub_error:
- self.report_warning("Some error while getting the subtitles")
- else:
- try:
- sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
- self.report_writesubtitles(sub_filename)
- with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
- subfile.write(sub)
- except (OSError, IOError):
- self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
- return
- if self.params.get('onlysubtitles', False):
- return
-
- if self.params.get('writeinfojson', False):
- infofn = filename + u'.info.json'
- self.report_writeinfojson(infofn)
- try:
- json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
- write_json_file(json_info_dict, encodeFilename(infofn))
- except (OSError, IOError):
- self.report_error(u'Cannot write metadata to JSON file ' + infofn)
- return
-
- if not self.params.get('skip_download', False):
- if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
- success = True
- else:
- try:
- success = self._do_download(filename, info_dict)
- except (OSError, IOError) as err:
- raise UnavailableVideoError()
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
- self.report_error(u'unable to download video data: %s' % str(err))
- return
- except (ContentTooShortError, ) as err:
- self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
- return
-
- if success:
- try:
- self.post_process(filename, info_dict)
- except (PostProcessingError) as err:
- self.report_error(u'postprocessing: %s' % str(err))
- return
-
- def download(self, url_list):
- """Download a given list of URLs."""
- if len(url_list) > 1 and self.fixed_template():
- raise SameFileError(self.params['outtmpl'])
-
- for url in url_list:
- try:
- #It also downloads the videos
- videos = self.extract_info(url)
- except UnavailableVideoError:
- self.trouble(u'\nERROR: unable to download video')
- except MaxDownloadsReached:
- self.to_screen(u'[info] Maximum number of downloaded files reached.')
- raise
-
- return self._download_retcode
-
- def post_process(self, filename, ie_info):
- """Run all the postprocessors on the given file."""
- info = dict(ie_info)
- info['filepath'] = filename
- keep_video = None
- for pp in self._pps:
- try:
- keep_video_wish,new_info = pp.run(info)
- if keep_video_wish is not None:
- if keep_video_wish:
- keep_video = keep_video_wish
- elif keep_video is None:
- # No clear decision yet, let IE decide
- keep_video = keep_video_wish
- except PostProcessingError as e:
- self.to_stderr(u'ERROR: ' + e.msg)
- if keep_video is False and not self.params.get('keepvideo', False):
- try:
- self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
- os.remove(encodeFilename(filename))
- except (IOError, OSError):
- self.report_warning(u'Unable to remove downloaded video file')
-
- def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path):