_ Git - youtube-dl/blob - devscripts/buildserver.py

   1 #!/usr/bin/python3
   2
   3 import argparse
   4 import ctypes
   5 import functools
   6 import shutil
   7 import subprocess
   8 import sys
   9 import tempfile
  10 import threading
  11 import traceback
  12 import os.path
  13
  14 sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
  15 from youtube_dl.compat import (
  16     compat_http_server,
  17     compat_str,
  18     compat_urlparse,
  19 )
  20
  21 # These are not used outside of buildserver.py thus not in compat.py
  22
  23 try:
  24     import winreg as compat_winreg
  25 except ImportError:  # Python 2
  26     import _winreg as compat_winreg
  27
  28 try:
  29     import socketserver as compat_socketserver
  30 except ImportError:  # Python 2
  31     import SocketServer as compat_socketserver
  32
  33 try:
  34     compat_input = raw_input
  35 except NameError:  # Python 3
  36     compat_input = input
  37
  38
  39 class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
  40     allow_reuse_address = True
  41
  42
  43 advapi32 = ctypes.windll.advapi32
  44
  45 SC_MANAGER_ALL_ACCESS = 0xf003f
  46 SC_MANAGER_CREATE_SERVICE = 0x02
  47 SERVICE_WIN32_OWN_PROCESS = 0x10
  48 SERVICE_AUTO_START = 0x2
  49 SERVICE_ERROR_NORMAL = 0x1
  50 DELETE = 0x00010000
  51 SERVICE_STATUS_START_PENDING = 0x00000002
  52 SERVICE_STATUS_RUNNING = 0x00000004
  53 SERVICE_ACCEPT_STOP = 0x1
  54
  55 SVCNAME = 'youtubedl_builder'
  56
  57 LPTSTR = ctypes.c_wchar_p
  58 START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
  59
  60
  61 class SERVICE_TABLE_ENTRY(ctypes.Structure):
  62     _fields_ = [
  63         ('lpServiceName', LPTSTR),
  64         ('lpServiceProc', START_CALLBACK)
  65     ]
  66
  67
  68 HandlerEx = ctypes.WINFUNCTYPE(
  69     ctypes.c_int,     # return
  70     ctypes.c_int,     # dwControl
  71     ctypes.c_int,     # dwEventType
  72     ctypes.c_void_p,  # lpEventData,
  73     ctypes.c_void_p,  # lpContext,
  74 )
  75
  76
  77 def _ctypes_array(c_type, py_array):
  78     ar = (c_type * len(py_array))()
  79     ar[:] = py_array
  80     return ar
  81
  82
  83 def win_OpenSCManager():
  84     res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
  85     if not res:
  86         raise Exception('Opening service manager failed - '
  87                         'are you running this as administrator?')
  88     return res
  89
  90
  91 def win_install_service(service_name, cmdline):
  92     manager = win_OpenSCManager()
  93     try:
  94         h = advapi32.CreateServiceW(
  95             manager, service_name, None,
  96             SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
  97             SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
  98             cmdline, None, None, None, None, None)
  99         if not h:
 100             raise OSError('Service creation failed: %s' % ctypes.FormatError())
 101
 102         advapi32.CloseServiceHandle(h)
 103     finally:
 104         advapi32.CloseServiceHandle(manager)
 105
 106
 107 def win_uninstall_service(service_name):
 108     manager = win_OpenSCManager()
 109     try:
 110         h = advapi32.OpenServiceW(manager, service_name, DELETE)
 111         if not h:
 112             raise OSError('Could not find service %s: %s' % (
 113                 service_name, ctypes.FormatError()))
 114
 115         try:
 116             if not advapi32.DeleteService(h):
 117                 raise OSError('Deletion failed: %s' % ctypes.FormatError())
 118         finally:
 119             advapi32.CloseServiceHandle(h)
 120     finally:
 121         advapi32.CloseServiceHandle(manager)
 122
 123
 124 def win_service_report_event(service_name, msg, is_error=True):
 125     with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
 126         f.write(msg + '\n')
 127
 128     event_log = advapi32.RegisterEventSourceW(None, service_name)
 129     if not event_log:
 130         raise OSError('Could not report event: %s' % ctypes.FormatError())
 131
 132     try:
 133         type_id = 0x0001 if is_error else 0x0004
 134         event_id = 0xc0000000 if is_error else 0x40000000
 135         lines = _ctypes_array(LPTSTR, [msg])
 136
 137         if not advapi32.ReportEventW(
 138                 event_log, type_id, 0, event_id, None, len(lines), 0,
 139                 lines, None):
 140             raise OSError('Event reporting failed: %s' % ctypes.FormatError())
 141     finally:
 142         advapi32.DeregisterEventSource(event_log)
 143
 144
 145 def win_service_handler(stop_event, *args):
 146     try:
 147         raise ValueError('Handler called with args ' + repr(args))
 148         TODO
 149     except Exception as e:
 150         tb = traceback.format_exc()
 151         msg = str(e) + '\n' + tb
 152         win_service_report_event(service_name, msg, is_error=True)
 153         raise
 154
 155
 156 def win_service_set_status(handle, status_code):
 157     svcStatus = SERVICE_STATUS()
 158     svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
 159     svcStatus.dwCurrentState = status_code
 160     svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
 161
 162     svcStatus.dwServiceSpecificExitCode = 0
 163
 164     if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
 165         raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
 166
 167
 168 def win_service_main(service_name, real_main, argc, argv_raw):
 169     try:
 170         # args = [argv_raw[i].value for i in range(argc)]
 171         stop_event = threading.Event()
 172         handler = HandlerEx(functools.partial(stop_event, win_service_handler))
 173         h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
 174         if not h:
 175             raise OSError('Handler registration failed: %s' %
 176                           ctypes.FormatError())
 177
 178         TODO
 179     except Exception as e:
 180         tb = traceback.format_exc()
 181         msg = str(e) + '\n' + tb
 182         win_service_report_event(service_name, msg, is_error=True)
 183         raise
 184
 185
 186 def win_service_start(service_name, real_main):
 187     try:
 188         cb = START_CALLBACK(
 189             functools.partial(win_service_main, service_name, real_main))
 190         dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
 191             SERVICE_TABLE_ENTRY(
 192                 service_name,
 193                 cb
 194             ),
 195             SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
 196         ])
 197
 198         if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
 199             raise OSError('ctypes start failed: %s' % ctypes.FormatError())
 200     except Exception as e:
 201         tb = traceback.format_exc()
 202         msg = str(e) + '\n' + tb
 203         win_service_report_event(service_name, msg, is_error=True)
 204         raise
 205
 206
 207 def main(args=None):
 208     parser = argparse.ArgumentParser()
 209     parser.add_argument('-i', '--install',
 210                         action='store_const', dest='action', const='install',
 211                         help='Launch at Windows startup')
 212     parser.add_argument('-u', '--uninstall',
 213                         action='store_const', dest='action', const='uninstall',
 214                         help='Remove Windows service')
 215     parser.add_argument('-s', '--service',
 216                         action='store_const', dest='action', const='service',
 217                         help='Run as a Windows service')
 218     parser.add_argument('-b', '--bind', metavar='<host:port>',
 219                         action='store', default='localhost:8142',
 220                         help='Bind to host:port (default %default)')
 221     options = parser.parse_args(args=args)
 222
 223     if options.action == 'install':
 224         fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
 225         cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
 226         win_install_service(SVCNAME, cmdline)
 227         return
 228
 229     if options.action == 'uninstall':
 230         win_uninstall_service(SVCNAME)
 231         return
 232
 233     if options.action == 'service':
 234         win_service_start(SVCNAME, main)
 235         return
 236
 237     host, port_str = options.bind.split(':')
 238     port = int(port_str)
 239
 240     print('Listening on %s:%d' % (host, port))
 241     srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
 242     thr = threading.Thread(target=srv.serve_forever)
 243     thr.start()
 244     compat_input('Press ENTER to shut down')
 245     srv.shutdown()
 246     thr.join()
 247
 248
 249 def rmtree(path):
 250     for name in os.listdir(path):
 251         fname = os.path.join(path, name)
 252         if os.path.isdir(fname):
 253             rmtree(fname)
 254         else:
 255             os.chmod(fname, 0o666)
 256             os.remove(fname)
 257     os.rmdir(path)
 258
 259
 260 class BuildError(Exception):
 261     def __init__(self, output, code=500):
 262         self.output = output
 263         self.code = code
 264
 265     def __str__(self):
 266         return self.output
 267
 268
 269 class HTTPError(BuildError):
 270     pass
 271
 272
 273 class PythonBuilder(object):
 274     def __init__(self, **kwargs):
 275         python_version = kwargs.pop('python', '3.4')
 276         python_path = None
 277         for node in ('Wow6432Node\\', ''):
 278             try:
 279                 key = compat_winreg.OpenKey(
 280                     compat_winreg.HKEY_LOCAL_MACHINE,
 281                     r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version))
 282                 try:
 283                     python_path, _ = compat_winreg.QueryValueEx(key, '')
 284                 finally:
 285                     compat_winreg.CloseKey(key)
 286                 break
 287             except Exception:
 288                 pass
 289
 290         if not python_path:
 291             raise BuildError('No such Python version: %s' % python_version)
 292
 293         self.pythonPath = python_path
 294
 295         super(PythonBuilder, self).__init__(**kwargs)
 296
 297
 298 class GITInfoBuilder(object):
 299     def __init__(self, **kwargs):
 300         try:
 301             self.user, self.repoName = kwargs['path'][:2]
 302             self.rev = kwargs.pop('rev')
 303         except ValueError:
 304             raise BuildError('Invalid path')
 305         except KeyError as e:
 306             raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
 307
 308         path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
 309         if not os.path.exists(path):
 310             os.makedirs(path)
 311         self.basePath = tempfile.mkdtemp(dir=path)
 312         self.buildPath = os.path.join(self.basePath, 'build')
 313
 314         super(GITInfoBuilder, self).__init__(**kwargs)
 315
 316
 317 class GITBuilder(GITInfoBuilder):
 318     def build(self):
 319         try:
 320             subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
 321             subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
 322         except subprocess.CalledProcessError as e:
 323             raise BuildError(e.output)
 324
 325         super(GITBuilder, self).build()
 326
 327
 328 class YoutubeDLBuilder(object):
 329     authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile']
 330
 331     def __init__(self, **kwargs):
 332         if self.repoName != 'youtube-dl':
 333             raise BuildError('Invalid repository "%s"' % self.repoName)
 334         if self.user not in self.authorizedUsers:
 335             raise HTTPError('Unauthorized user "%s"' % self.user, 401)
 336
 337         super(YoutubeDLBuilder, self).__init__(**kwargs)
 338
 339     def build(self):
 340         try:
 341             proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath)
 342             proc.wait()
 343             #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
 344             #                        cwd=self.buildPath)
 345         except subprocess.CalledProcessError as e:
 346             raise BuildError(e.output)
 347
 348         super(YoutubeDLBuilder, self).build()
 349
 350
 351 class DownloadBuilder(object):
 352     def __init__(self, **kwargs):
 353         self.handler = kwargs.pop('handler')
 354         self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
 355         self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
 356         if not self.srcPath.startswith(self.buildPath):
 357             raise HTTPError(self.srcPath, 401)
 358
 359         super(DownloadBuilder, self).__init__(**kwargs)
 360
 361     def build(self):
 362         if not os.path.exists(self.srcPath):
 363             raise HTTPError('No such file', 404)
 364         if os.path.isdir(self.srcPath):
 365             raise HTTPError('Is a directory: %s' % self.srcPath, 401)
 366
 367         self.handler.send_response(200)
 368         self.handler.send_header('Content-Type', 'application/octet-stream')
 369         self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
 370         self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
 371         self.handler.end_headers()
 372
 373         with open(self.srcPath, 'rb') as src:
 374             shutil.copyfileobj(src, self.handler.wfile)
 375
 376         super(DownloadBuilder, self).build()
 377
 378
 379 class CleanupTempDir(object):
 380     def build(self):
 381         try:
 382             rmtree(self.basePath)
 383         except Exception as e:
 384             print('WARNING deleting "%s": %s' % (self.basePath, e))
 385
 386         super(CleanupTempDir, self).build()
 387
 388
 389 class Null(object):
 390     def __init__(self, **kwargs):
 391         pass
 392
 393     def start(self):
 394         pass
 395
 396     def close(self):
 397         pass
 398
 399     def build(self):
 400         pass
 401
 402
 403 class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
 404     pass
 405
 406
 407 class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler):
 408     actionDict = {'build': Builder, 'download': Builder}  # They're the same, no more caching.
 409
 410     def do_GET(self):
 411         path = compat_urlparse.urlparse(self.path)
 412         paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()])
 413         action, _, path = path.path.strip('/').partition('/')
 414         if path:
 415             path = path.split('/')
 416             if action in self.actionDict:
 417                 try:
 418                     builder = self.actionDict[action](path=path, handler=self, **paramDict)
 419                     builder.start()
 420                     try:
 421                         builder.build()
 422                     finally:
 423                         builder.close()
 424                 except BuildError as e:
 425                     self.send_response(e.code)
 426                     msg = compat_str(e).encode('UTF-8')
 427                     self.send_header('Content-Type', 'text/plain; charset=UTF-8')
 428                     self.send_header('Content-Length', len(msg))
 429                     self.end_headers()
 430                     self.wfile.write(msg)
 431                 except HTTPError as e:
 432                     self.send_response(e.code, str(e))
 433             else:
 434                 self.send_response(500, 'Unknown build method "%s"' % action)
 435         else:
 436             self.send_response(500, 'Malformed URL')
 437
 438 if __name__ == '__main__':
 439     main()