From: Philipp Hagemeister Date: Sat, 23 Aug 2014 14:41:14 +0000 (+0200) Subject: Merge remote-tracking branch 'sehrgut/Grooveshark' X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=commitdiff_plain;h=00558d94145f97c644e66ec086fa9b9d8c58280f;hp=-c;p=youtube-dl Merge remote-tracking branch 'sehrgut/Grooveshark' Conflicts: youtube_dl/__init__.py youtube_dl/extractor/__init__.py --- 00558d94145f97c644e66ec086fa9b9d8c58280f diff --combined youtube_dl/__init__.py index f156ba3a0,cd4e9b484..613ce3576 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@@ -56,32 -56,19 +56,33 @@@ __authors__ = 'Nicolas Évrard', 'Jason Normore', 'Hoje Lee', - 'Keith Beckman' + 'Adam Thalhammer', + 'Georg Jähnig', + 'Ralf Haring', + 'Koki Takahashi', + 'Ariset Llerena', + 'Adam Malcontenti-Wilson', + 'Tobias Bell', + 'Naglis Jonaitis', + 'Charles Chen', + 'Hassaan Ali', + 'Dobrosław Żybort', + 'David Fabijan', + 'Sebastian Haas', + 'Alexander Kirk', + 'Erik Johnson', ++ 'Keith Beckman', ) __license__ = 'Public Domain' import codecs import io -import locale import optparse import os import random -import re import shlex +import shutil import sys @@@ -103,7 -90,7 +104,7 @@@ from .utils import write_string, ) from .update import update_self -from .FileDownloader import ( +from .downloader import ( FileDownloader, ) from .extractor import gen_extractors @@@ -228,7 -215,6 +229,7 @@@ def parseOpts(overrideArguments=None) downloader = optparse.OptionGroup(parser, 'Download Options') postproc = optparse.OptionGroup(parser, 'Post-processing Options') filesystem = optparse.OptionGroup(parser, 'Filesystem Options') + workarounds = optparse.OptionGroup(parser, 'Workarounds') verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') general.add_option('-h', '--help', @@@ -245,6 -231,14 +246,6 @@@ general.add_option('--dump-user-agent', action='store_true', dest='dump_user_agent', help='display the current browser identification', default=False) - general.add_option('--user-agent', - dest='user_agent', help='specify a custom user agent', metavar='UA') - general.add_option('--referer', - dest='referer', help='specify a custom referer, use if the video access is restricted to one domain', - metavar='REF', default=None) - general.add_option('--add-header', - dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append", - metavar='FIELD:VALUE') general.add_option('--list-extractors', action='store_true', dest='list_extractors', help='List all supported extractors and the URLs they would handle', default=False) @@@ -254,17 -248,33 +255,17 @@@ general.add_option( '--proxy', dest='proxy', default=None, metavar='URL', help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection') - general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') - general.add_option( - '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure', - help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)') - general.add_option( - '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR', - help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') - general.add_option( - '--no-cache-dir', action='store_const', const=None, dest='cachedir', - help='Disable filesystem caching') general.add_option( '--socket-timeout', dest='socket_timeout', type=float, default=None, help=u'Time to wait before giving up, in seconds') - general.add_option( - '--bidi-workaround', dest='bidi_workaround', action='store_true', - help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') general.add_option( '--default-search', dest='default_search', metavar='PREFIX', - help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.') + help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.') general.add_option( '--ignore-config', action='store_true', help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)') - general.add_option( - '--encoding', dest='encoding', metavar='ENCODING', - help='Force the specified encoding (experimental)') selection.add_option( '--playlist-start', @@@ -365,33 -375,6 +366,33 @@@ help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) + workarounds.add_option( + '--encoding', dest='encoding', metavar='ENCODING', + help='Force the specified encoding (experimental)') + workarounds.add_option( + '--no-check-certificate', action='store_true', + dest='no_check_certificate', default=False, + help='Suppress HTTPS certificate validation.') + workarounds.add_option( + '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure', + help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)') + workarounds.add_option( + '--user-agent', metavar='UA', + dest='user_agent', help='specify a custom user agent') + workarounds.add_option( + '--referer', metavar='REF', + dest='referer', default=None, + help='specify a custom referer, use if the video access is restricted to one domain', + ) + workarounds.add_option( + '--add-header', metavar='FIELD:VALUE', + dest='headers', action='append', + help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', + ) + workarounds.add_option( + '--bidi-workaround', dest='bidi_workaround', action='store_true', + help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') + verbosity.add_option('-q', '--quiet', action='store_true', dest='quiet', help='activates quiet mode', default=False) verbosity.add_option( @@@ -449,10 -432,12 +450,10 @@@ help='Display sent and read HTTP traffic') - filesystem.add_option('-t', '--title', - action='store_true', dest='usetitle', help='use title in file name (default)', default=False) + filesystem.add_option('-a', '--batch-file', + dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') filesystem.add_option('--id', action='store_true', dest='useid', help='use only video ID in file name', default=False) - filesystem.add_option('-l', '--literal', - action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) filesystem.add_option('-A', '--auto-number', action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False) @@@ -478,10 -463,11 +479,10 @@@ filesystem.add_option('--restrict-filenames', action='store_true', dest='restrictfilenames', help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) - filesystem.add_option('-a', '--batch-file', - dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') - filesystem.add_option('--load-info', - dest='load_info_filename', metavar='FILE', - help='json file containing the video information (created with the "--write-json" option)') + filesystem.add_option('-t', '--title', + action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False) + filesystem.add_option('-l', '--literal', + action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) filesystem.add_option('-w', '--no-overwrites', action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) filesystem.add_option('-c', '--continue', @@@ -489,6 -475,8 +490,6 @@@ filesystem.add_option('--no-continue', action='store_false', dest='continue_dl', help='do not resume partially downloaded files (restart from beginning)') - filesystem.add_option('--cookies', - dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') filesystem.add_option('--no-part', action='store_true', dest='nopart', help='do not use .part files', default=False) filesystem.add_option('--no-mtime', @@@ -506,20 -494,6 +507,20 @@@ filesystem.add_option('--write-thumbnail', action='store_true', dest='writethumbnail', help='write thumbnail image to disk', default=False) + filesystem.add_option('--load-info', + dest='load_info_filename', metavar='FILE', + help='json file containing the video information (created with the "--write-json" option)') + filesystem.add_option('--cookies', + dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') + filesystem.add_option( + '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR', + help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') + filesystem.add_option( + '--no-cache-dir', action='store_const', const=None, dest='cachedir', + help='Disable filesystem caching') + filesystem.add_option( + '--rm-cache-dir', action='store_true', dest='rm_cachedir', + help='Delete all filesystem cache files') postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, @@@ -529,7 -503,7 +530,7 @@@ postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None, - help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)') + help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)') postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, help='keeps the video file on disk after the post-processing; the video is erased by default') postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, @@@ -553,7 -527,6 +554,7 @@@ parser.add_option_group(downloader) parser.add_option_group(filesystem) parser.add_option_group(verbosity) + parser.add_option_group(workarounds) parser.add_option_group(video_format) parser.add_option_group(subtitles) parser.add_option_group(authentication) @@@ -653,7 -626,7 +654,7 @@@ def _real_main(argv=None) if desc is False: continue if hasattr(ie, 'SEARCH_KEY'): - _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise') + _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny') _COUNTS = (u'', u'5', u'10', u'all') desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) compat_print(desc) @@@ -714,7 -687,7 +715,7 @@@ date = DateRange.day(opts.date) else: date = DateRange(opts.dateafter, opts.datebefore) - if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search: + if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search: parser.error(u'--default-search invalid; did you forget a colon (:) at the end?') # Do not download videos when there are audio-only formats @@@ -853,26 -826,9 +854,26 @@@ if opts.update_self: update_self(ydl.to_screen, opts.verbose) + # Remove cache dir + if opts.rm_cachedir: + if opts.cachedir is None: + ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)') + else: + if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir): + ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir') + retcode = 141 + else: + ydl.to_screen( + u'Removing cache dir %s .' % opts.cachedir, + skip_eol=True) + if os.path.exists(opts.cachedir): + ydl.to_screen(u'.', skip_eol=True) + shutil.rmtree(opts.cachedir) + ydl.to_screen(u'.') + # Maybe do nothing if (len(all_urls) < 1) and (opts.load_info_filename is None): - if not opts.update_self: + if not (opts.update_self or opts.rm_cachedir): parser.error(u'you must provide at least one URL') else: sys.exit() diff --combined youtube_dl/extractor/__init__.py index ddcf2e414,3f6c67fa5..0d7824023 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@@ -1,11 -1,8 +1,11 @@@ +from .abc import ABCIE from .academicearth import AcademicEarthCourseIE from .addanime import AddAnimeIE +from .adultswim import AdultSwimIE from .aftonbladet import AftonbladetIE from .anitube import AnitubeIE from .aol import AolIE +from .allocine import AllocineIE from .aparat import AparatIE from .appletrailers import AppleTrailersIE from .archiveorg import ArchiveOrgIE @@@ -54,7 -51,6 +54,7 @@@ from .cnn import from .collegehumor import CollegeHumorIE from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE from .condenast import CondeNastIE +from .cracked import CrackedIE from .criterion import CriterionIE from .crunchyroll import CrunchyrollIE from .cspan import CSpanIE @@@ -65,11 -61,8 +65,11 @@@ from .dailymotion import DailymotionUserIE, ) from .daum import DaumIE +from .dfb import DFBIE from .dotsub import DotsubIE from .dreisat import DreiSatIE +from .drtv import DRTVIE +from .dump import DumpIE from .defense import DefenseGouvFrIE from .discovery import DiscoveryIE from .divxstage import DivxStageIE @@@ -78,10 -71,6 +78,10 @@@ from .ebaumsworld import EbaumsWorldI from .ehow import EHowIE from .eighttracks import EightTracksIE from .eitb import EitbIE +from .ellentv import ( + EllenTVIE, + EllenTVClipsIE, +) from .elpais import ElPaisIE from .empflix import EmpflixIE from .engadget import EngadgetIE @@@ -92,7 -81,6 +92,7 @@@ from .extremetube import ExtremeTubeI from .facebook import FacebookIE from .faz import FazIE from .fc2 import FC2IE +from .firedrive import FiredriveIE from .firstpost import FirstpostIE from .firsttv import FirstTVIE from .fivemin import FiveMinIE @@@ -115,23 -103,18 +115,24 @@@ from .freesound import FreesoundI from .freespeech import FreespeechIE from .funnyordie import FunnyOrDieIE from .gamekings import GamekingsIE +from .gameone import GameOneIE from .gamespot import GameSpotIE +from .gamestar import GameStarIE from .gametrailers import GametrailersIE from .gdcvault import GDCVaultIE from .generic import GenericIE +from .godtube import GodTubeIE from .googleplus import GooglePlusIE from .googlesearch import GoogleSearchIE +from .gorillavid import GorillaVidIE +from .goshgay import GoshgayIE + from .grooveshark import GroovesharkIE from .hark import HarkIE from .helsinki import HelsinkiIE from .hentaistigma import HentaiStigmaIE from .hotnewhiphop import HotNewHipHopIE from .howcast import HowcastIE +from .howstuffworks import HowStuffWorksIE from .huffpost import HuffPostIE from .hypem import HypemIE from .iconosquare import IconosquareIE @@@ -149,10 -132,8 +150,10 @@@ from .ivi import IviIE, IviCompilationIE ) +from .izlesene import IzleseneIE from .jadorecettepub import JadoreCettePubIE from .jeuxvideo import JeuxVideoIE +from .jove import JoveIE from .jukebox import JukeboxIE from .justintv import JustinTVIE from .jpopsukitv import JpopsukiIE @@@ -162,16 -143,10 +163,16 @@@ from .khanacademy import KhanAcademyI from .kickstarter import KickStarterIE from .keek import KeekIE from .kontrtube import KontrTubeIE +from .krasview import KrasViewIE +from .ku6 import Ku6IE from .la7 import LA7IE from .lifenews import LifeNewsIE from .liveleak import LiveLeakIE -from .livestream import LivestreamIE, LivestreamOriginalIE +from .livestream import ( + LivestreamIE, + LivestreamOriginalIE, + LivestreamShortenerIE, +) from .lynda import ( LyndaIE, LyndaCourseIE @@@ -184,22 -159,16 +185,22 @@@ from .mdr import MDRI from .metacafe import MetacafeIE from .metacritic import MetacriticIE from .mit import TechTVMITIE, MITIE, OCWMITIE +from .mitele import MiTeleIE from .mixcloud import MixcloudIE +from .mlb import MLBIE from .mpora import MporaIE from .mofosex import MofosexIE +from .mojvideo import MojvideoIE from .mooshare import MooshareIE from .morningstar import MorningstarIE +from .motherless import MotherlessIE from .motorsport import MotorsportIE +from .movieclips import MovieClipsIE from .moviezine import MoviezineIE from .movshare import MovShareIE from .mtv import ( MTVIE, + MTVServicesEmbeddedIE, MTVIggyIE, ) from .musicplayon import MusicPlayOnIE @@@ -226,25 -195,16 +227,25 @@@ from .normalboots import NormalbootsI from .novamov import NovaMovIE from .nowness import NownessIE from .nowvideo import NowVideoIE -from .nrk import NRKIE +from .npo import NPOIE +from .nrk import ( + NRKIE, + NRKTVIE, +) from .ntv import NTVIE from .nytimes import NYTimesIE from .nuvid import NuvidIE -from .oe1 import OE1IE from .ooyala import OoyalaIE -from .orf import ORFIE +from .orf import ( + ORFTVthekIE, + ORFOE1IE, + ORFFM4IE, +) from .parliamentliveuk import ParliamentLiveUKIE +from .patreon import PatreonIE from .pbs import PBSIE from .photobucket import PhotobucketIE +from .playfm import PlayFMIE from .playvid import PlayvidIE from .podomatic import PodomaticIE from .pornhd import PornHdIE @@@ -253,20 -213,16 +254,20 @@@ from .pornotube import PornotubeI from .prosiebensat1 import ProSiebenSat1IE from .pyvideo import PyvideoIE from .radiofrance import RadioFranceIE +from .rai import RaiIE from .rbmaradio import RBMARadioIE from .redtube import RedTubeIE +from .reverbnation import ReverbNationIE from .ringtv import RingTVIE from .ro220 import Ro220IE from .rottentomatoes import RottenTomatoesIE from .roxwel import RoxwelIE from .rtbf import RTBFIE +from .rtlnl import RtlXlIE from .rtlnow import RTLnowIE from .rts import RTSIE -from .rtve import RTVEALaCartaIE +from .rtve import RTVEALaCartaIE, RTVELiveIE +from .ruhd import RUHDIE from .rutube import ( RutubeIE, RutubeChannelIE, @@@ -274,13 -230,9 +275,13 @@@ RutubePersonIE, ) from .rutv import RUTVIE +from .sapo import SapoIE from .savefrom import SaveFromIE +from .sbs import SBSIE from .scivee import SciVeeIE +from .screencast import ScreencastIE from .servingsys import ServingSysIE +from .shared import SharedIE from .sina import SinaIE from .slideshare import SlideshareIE from .slutload import SlutloadIE @@@ -290,8 -242,6 +291,8 @@@ from .smotri import SmotriUserIE, SmotriBroadcastIE, ) +from .snotr import SnotrIE +from .sockshare import SockshareIE from .sohu import SohuIE from .soundcloud import ( SoundcloudIE, @@@ -299,33 -249,23 +300,33 @@@ SoundcloudUserIE, SoundcloudPlaylistIE ) -from .southparkstudios import ( - SouthParkStudiosIE, +from .soundgasm import SoundgasmIE +from .southpark import ( + SouthParkIE, SouthparkDeIE, ) from .space import SpaceIE from .spankwire import SpankwireIE from .spiegel import SpiegelIE +from .spiegeltv import SpiegeltvIE from .spike import SpikeIE from .stanfordoc import StanfordOpenClassroomIE from .steam import SteamIE from .streamcloud import StreamcloudIE from .streamcz import StreamCZIE +from .swrmediathek import SWRMediathekIE from .syfy import SyfyIE from .sztvhu import SztvHuIE +from .tagesschau import TagesschauIE +from .teachertube import ( + TeacherTubeIE, + TeacherTubeUserIE, +) +from .teachingchannel import TeachingChannelIE from .teamcoco import TeamcocoIE from .techtalks import TechTalksIE from .ted import TEDIE +from .tenplay import TenPlayIE from .testurl import TestURLIE from .tf1 import TF1IE from .theplatform import ThePlatformIE @@@ -343,8 -283,6 +344,8 @@@ from .tumblr import TumblrI from .tutv import TutvIE from .tvigle import TvigleIE from .tvp import TvpIE +from .tvplay import TVPlayIE +from .ubu import UbuIE from .udemy import ( UdemyIE, UdemyCourseIE @@@ -357,7 -295,6 +358,7 @@@ from .veehd import VeeHDI from .veoh import VeohIE from .vesti import VestiIE from .vevo import VevoIE +from .vh1 import VH1IE from .viddler import ViddlerIE from .videobam import VideoBamIE from .videodetective import VideoDetectiveIE @@@ -366,7 -303,6 +367,7 @@@ from .videofyme import VideofyMeI from .videopremium import VideoPremiumIE from .videott import VideoTtIE from .videoweed import VideoWeedIE +from .vidme import VidmeIE from .vimeo import ( VimeoIE, VimeoChannelIE, @@@ -376,17 -312,14 +377,17 @@@ VimeoReviewIE, VimeoWatchLaterIE, ) +from .vimple import VimpleIE from .vine import ( VineIE, VineUserIE, ) from .viki import VikiIE from .vk import VKIE +from .vodlocker import VodlockerIE from .vube import VubeIE from .vuclip import VuClipIE +from .vulture import VultureIE from .washingtonpost import WashingtonPostIE from .wat import WatIE from .wdr import ( @@@ -398,9 -331,7 +399,9 @@@ from .weibo import WeiboI from .wimp import WimpIE from .wistia import WistiaIE from .worldstarhiphop import WorldStarHipHopIE +from .wrzuta import WrzutaIE from .xbef import XBefIE +from .xboxclips import XboxClipsIE from .xhamster import XHamsterIE from .xnxx import XNXXIE from .xvideos import XVideosIE @@@ -430,7 -361,6 +431,7 @@@ from .youtube import YoutubeUserIE, YoutubeWatchLaterIE, ) + from .zdf import ZDFIE