From: Philipp Hagemeister Date: Sun, 24 Aug 2014 05:14:23 +0000 (+0200) Subject: Merge remote-tracking branch 'riking/twofactor' X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=commitdiff_plain;h=9480d1a56674f95f135562d2133cbf12f6a96bbc;hp=-c;p=youtube-dl Merge remote-tracking branch 'riking/twofactor' --- 9480d1a56674f95f135562d2133cbf12f6a96bbc diff --combined youtube_dl/__init__.py index d24157ba6,80de211e7..a96bf9b5c --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@@ -69,10 -69,6 +69,10 @@@ __authors__ = 'Dobrosław Żybort', 'David Fabijan', 'Sebastian Haas', + 'Alexander Kirk', + 'Erik Johnson', + 'Keith Beckman', + 'Ole Ernst', ) __license__ = 'Public Domain' @@@ -318,6 -314,8 +318,8 @@@ def parseOpts(overrideArguments=None) dest='username', metavar='USERNAME', help='account username') authentication.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password') + authentication.add_option('-2', '--twofactor', + dest='twofactor', metavar='TWOFACTOR', help='two-factor auth code') authentication.add_option('-n', '--netrc', action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) authentication.add_option('--video-password', @@@ -752,6 -750,7 +754,7 @@@ def _real_main(argv=None) 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, + 'twofactor': opts.twofactor, 'videopassword': opts.videopassword, 'quiet': (opts.quiet or any_printing), 'no_warnings': opts.no_warnings, diff --combined youtube_dl/extractor/common.py index 9d85a538c,45a17f8ad..4d5b48167 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@@ -84,12 -84,6 +84,12 @@@ class InfoExtractor(object) format, irrespective of the file format. -1 for default (order by other properties), -2 or smaller for less than default. + * http_referer HTTP Referer header value to set. + * http_method HTTP method to use for the download. + * http_headers A dictionary of additional HTTP headers + to add to the request. + * http_post_data Additional data to send with a POST + request. url: Final video URL. ext: Video filename extension. format: The video format, defaults to ext (used for --get-format) @@@ -440,6 -434,22 +440,22 @@@ return (username, password) + def _get_tfa_info(self): + """ + Get the two-factor authentication info + TODO - asking the user will be required for sms/phone verify + currently just uses the command line option + If there's no info available, return None + """ + if self._downloader is None: + return None + downloader_params = self._downloader.params + + if downloader_params.get('twofactor', None) is not None: + return downloader_params['twofactor'] + + return None + # Helper functions for extracting OpenGraph info @staticmethod def _og_regexes(prop): @@@ -469,9 -479,8 +485,9 @@@ return self._og_search_property('title', html, **kargs) def _og_search_video_url(self, html, name='video url', secure=True, **kargs): - regexes = self._og_regexes('video') - if secure: regexes = self._og_regexes('video:secure_url') + regexes + regexes = self._og_regexes('video') + self._og_regexes('video:url') + if secure: + regexes = self._og_regexes('video:secure_url') + regexes return self._html_search_regex(regexes, html, name, **kargs) def _og_search_url(self, html, **kargs): diff --combined youtube_dl/extractor/youtube.py index 225e2b7f4,5bfe5e7e5..75044d71a --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@@ -37,6 -37,7 +37,7 @@@ from ..utils import class YoutubeBaseInfoExtractor(InfoExtractor): """Provide base functions for Youtube extractors""" _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' + _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor' _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' _AGE_URL = 'https://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' _NETRC_MACHINE = 'youtube' @@@ -50,12 -51,19 +51,19 @@@ fatal=False)) def _login(self): + """ + Attempt to log in to YouTube. + True is returned if successful or skipped. + False is returned if login failed. + + If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised. + """ (username, password) = self._get_login_info() # No authentication to be performed if username is None: if self._LOGIN_REQUIRED: raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True) - return False + return True login_page = self._download_webpage( self._LOGIN_URL, None, @@@ -73,6 -81,7 +81,7 @@@ u'Email': username, u'GALX': galx, u'Passwd': password, + u'PersistentCookie': u'yes', u'_utf8': u'霱', u'bgresponse': u'js_disabled', @@@ -88,6 -97,7 +97,7 @@@ u'uilel': u'3', u'hl': u'en_US', } + # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode # chokes on unicode login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items()) @@@ -99,6 -109,68 +109,68 @@@ note=u'Logging in', errnote=u'unable to log in', fatal=False) if login_results is False: return False + + if re.search(r'id="errormsg_0_Passwd"', login_results) is not None: + raise ExtractorError(u'Please use your account password and a two-factor code instead of an application-specific password.', expected=True) + + # Two-Factor + # TODO add SMS and phone call support - these require making a request and then prompting the user + + if re.search(r'(?i)]* id="gaia_secondfactorform"', login_results) is not None: + tfa_code = self._get_tfa_info() + + if tfa_code is None: + self._downloader.report_warning(u'Two-factor authentication required. Provide it with --twofactor ') + self._downloader.report_warning(u'(Note that only TOTP (Google Authenticator App) codes work at this time.)') + return False + + # Unlike the first login form, secTok and timeStmp are both required for the TFA form + + match = re.search(r'id="secTok"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U) + if match is None: + self._downloader.report_warning(u'Failed to get secTok - did the page structure change?') + secTok = match.group(1) + match = re.search(r'id="timeStmp"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U) + if match is None: + self._downloader.report_warning(u'Failed to get timeStmp - did the page structure change?') + timeStmp = match.group(1) + + tfa_form_strs = { + u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1', + u'smsToken': u'', + u'smsUserPin': tfa_code, + u'smsVerifyPin': u'Verify', + + u'PersistentCookie': u'yes', + u'checkConnection': u'', + u'checkedDomains': u'youtube', + u'pstMsg': u'1', + u'secTok': secTok, + u'timeStmp': timeStmp, + u'service': u'youtube', + u'hl': u'en_US', + } + tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in tfa_form_strs.items()) + tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii') + + tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data) + tfa_results = self._download_webpage( + tfa_req, None, + note=u'Submitting TFA code', errnote=u'unable to submit tfa', fatal=False) + + if tfa_results is False: + return False + + if re.search(r'(?i)]* id="gaia_secondfactorform"', tfa_results) is not None: + self._downloader.report_warning(u'Two-factor code expired. Please try again, or use a one-use backup code instead.') + return False + if re.search(r'(?i)]* id="gaia_loginform"', tfa_results) is not None: + self._downloader.report_warning(u'unable to log in - did the page structure change?') + return False + if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None: + self._downloader.report_warning(u'Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.') + return False + if re.search(r'(?i)]* id="gaia_loginform"', login_results) is not None: self._downloader.report_warning(u'unable to log in: bad username or password') return False @@@ -225,7 -297,7 +297,7 @@@ class YoutubeIE(YoutubeBaseInfoExtracto '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40}, # Dash webm audio - '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 48, 'preference': -50}, + '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50}, '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50}, # RTMP (unnamed) @@@ -374,13 -446,6 +446,13 @@@ return lambda s: u''.join(s[i] for i in cache_spec) except IOError: pass # No cache available + except ValueError: + try: + file_size = os.path.getsize(cache_fn) + except (OSError, IOError) as oe: + file_size = str(oe) + self._downloader.report_warning( + u'Cache %s failed (%s)' % (cache_fn, file_size)) if player_type == 'js': code = self._download_webpage( @@@ -508,8 -573,6 +580,8 @@@ sub_lang_list = {} for l in lang_list: lang = l[1] + if lang in sub_lang_list: + continue params = compat_urllib_parse.urlencode({ 'lang': lang, 'v': video_id,