Create mach bootstrap based on Mozilla's mozboot bootstrapper

python/servo/bootstrap_commands.py

@@ -19,6 +19,7 @@ import shutil
 import sys
 import StringIO
 import tarfile
+import zipfile
 import urllib2
 
 from mach.decorators import (
@@ -101,7 +102,10 @@ def download_bytes(desc, src):
 
 
 def extract(src, dst, movedir=None):
-    tarfile.open(src).extractall(dst)
+    if src.endswith(".zip"):
+        zipfile.ZipFile(src).extractall(dst)
+    else:
+        tarfile.open(src).extractall(dst)
 
     if movedir:
         for f in os.listdir(movedir):
@@ -126,6 +130,24 @@ class MachCommands(CommandBase):
         else:
             print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])
 
+    @Command('bootstrap',
+             description='Install required packages for building.',
+             category='bootstrap')
+    @CommandArgument('--interactive', "-i",
+                     action='store_true',
+                     help='Need to answer any (Y/n) interactive prompts.')
+    @CommandArgument('--android',
+                     action='store_true',
+                     help='Install required packages for Android')
+    @CommandArgument('--force', '-f',
+                     action='store_true',
+                     help='Force reinstall packages')
+    def bootstrap(self, android=False, interactive=False, force=False):
+        from servo.bootstrapper.bootstrap import Bootstrapper
+
+        bootstrapper = Bootstrapper()
+        bootstrapper.bootstrap(android=android, interactive=interactive, force=force)
+
     @Command('bootstrap-rust',
              description='Download the Rust compiler',
              category='bootstrap')

python/servo/bootstrapper/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.

python/servo/bootstrapper/base.py

@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import distutils
+import subprocess
+
+
+class BaseBootstrapper(object):
+    """Base class for system bootstrappers."""
+
+    def __init__(self, interactive=False):
+        self.package_manager_updated = False
+        self.interactive = interactive
+
+    def ensure_system_packages(self):
+        '''
+        Check for missing packages.
+        '''
+        raise NotImplementedError('%s must implement ensure_system_packages()' %
+                                  __name__)
+
+    def install_system_packages(self):
+        '''
+        Install packages required to build Servo.
+        '''
+        raise NotImplementedError('%s must implement install_system_packages()' %
+                                  __name__)
+
+    def install_mobile_android_packages(self):
+        '''
+        Install packages required to build Servo for Android.
+        '''
+        raise NotImplementedError('Cannot bootstrap Servo for Android: '
+                                  '%s does not yet implement install_mobile_android_packages()'
+                                  % __name__)
+
+    def which(self, name):
+        """Python implementation of which.
+
+        It returns the path of an executable or None if it couldn't be found.
+        """
+        return distutils.spawn.find_executable(name)
+
+    def check_output(self, *args, **kwargs):
+        """Run subprocess.check_output."""
+        return subprocess.check_output(*args, **kwargs)
+
+    def _ensure_package_manager_updated(self):
+        if self.package_manager_updated:
+            return
+
+        self._update_package_manager()
+        self.package_manager_updated = True
+
+    def _update_package_manager(self):
+        """Updates the package manager's manifests/package list.
+
+        This should be defined in child classes.
+        """

python/servo/bootstrapper/bootstrap.py

@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+
+import sys
+
+from windows_gnu import WindowsGnuBootstrapper
+from windows_msvc import WindowsMsvcBootstrapper
+
+
+class Bootstrapper(object):
+    """Main class that performs system bootstrap."""
+
+    def __init__(self):
+        self.instance = None
+        cls = None
+        args = {}
+
+        if sys.platform.startswith('msys'):
+            cls = WindowsGnuBootstrapper
+
+        elif sys.platform.startswith('win32'):
+            cls = WindowsMsvcBootstrapper
+
+        if cls is None:
+            sys.exit('Bootstrap support is not yet available for your OS.')
+
+        self.instance = cls(**args)
+
+    def bootstrap(self, android=False, interactive=False, force=False):
+        self.instance.interactive = interactive
+        self.instance.force = force
+
+        if android:
+            self.instance.install_mobile_android_packages()
+        elif force:
+            self.instance.install_system_packages()
+        else:
+            self.instance.ensure_system_packages()

python/servo/bootstrapper/packages.py

@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Listed all packages for different platforms in one file
+
+WINDOWS_GNU = [
+    "mingw-w64-x86_64-toolchain",
+    "mingw-w64-x86_64-freetype",
+    "mingw-w64-x86_64-icu",
+    "mingw-w64-x86_64-nspr",
+    "mingw-w64-x86_64-ca-certificates",
+    "mingw-w64-x86_64-expat",
+    "mingw-w64-x86_64-cmake",
+    "tar",
+    "diffutils",
+    "patch",
+    "patchutils",
+    "make",
+    "python2-setuptools",
+]
+
+WINDOWS_MSVC = [
+    "cmake-3.6.1",
+    "ninja-1.7.1",
+    "openssl-1.0.1t-vs2015",
+    "moztools-0.0.1-5",
+]

python/servo/bootstrapper/windows_gnu.py

@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import subprocess
+
+from base import BaseBootstrapper
+from packages import WINDOWS_GNU as deps
+
+
+class WindowsGnuBootstrapper(BaseBootstrapper):
+    '''Bootstrapper for msys2 based environments for building in Windows.'''
+
+    def __init__(self, **kwargs):
+        BaseBootstrapper.__init__(self, **kwargs)
+
+        if not self.which('pacman'):
+            raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at '
+                                      'http://msys2.github.io/')
+
+    def ensure_system_packages(self):
+        install_packages = []
+        for p in deps:
+            command = ['pacman', '-Qs', p]
+            if self.run_check(command):
+                install_packages += [p]
+        if install_packages:
+            install_packages(install_packages)
+
+    def install_system_packages(self, packages=deps):
+        self._ensure_package_manager_updated()
+        self.pacman_install(*packages)
+
+    def install_mobile_android_packages(self):
+        sys.exit('We do not support building Android on Windows. Sorry!')
+
+    def _update_package_manager(self):
+        self.pacman_update()
+
+    def run(self, command):
+        subprocess.check_call(command, stdin=sys.stdin)
+
+    def run_check(self, command):
+        return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    def pacman_update(self):
+        command = ['pacman', '--sync', '--refresh']
+        self.run(command)
+
+    def pacman_upgrade(self):
+        command = ['pacman', '--sync', '--refresh', '--sysupgrade']
+        self.run(command)
+
+    def pacman_install(self, *packages):
+        command = ['pacman', '--sync']
+        if not self.force:
+            command.append('--needed')
+        if not self.interactive:
+            command.append('--noconfirm')
+        command.extend(packages)
+        self.run(command)
+
+        # downgrade GCC to 5.4.0-1
+        gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
+        gcc_version = "5.4.0-1"
+        mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
+        gcc_list = []
+        for gcc in gcc_type:
+            gcc_list += [mingw_url.format(gcc, gcc_version)]
+        downgrade_command = ['pacman', '-U']
+        if not self.interactive:
+            downgrade_command.append('--noconfirm')
+        downgrade_command.extend(gcc_list)
+        self.run(downgrade_command)

python/servo/bootstrapper/windows_msvc.py

@@ -0,0 +1,86 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import shutil
+from distutils import spawn
+
+from base import BaseBootstrapper
+from packages import WINDOWS_MSVC as deps
+
+
+class WindowsMsvcBootstrapper(BaseBootstrapper):
+    '''Bootstrapper for MSVC building on Windows.'''
+
+    def __init__(self, **kwargs):
+        BaseBootstrapper.__init__(self, **kwargs)
+
+    def ensure_system_packages(self):
+        self.install_system_packages()
+
+    def install_system_packages(self, packages=deps):
+        from servo.bootstrap_commands import extract, download_file
+
+        deps_dir = os.path.join(".servo", "msvc-dependencies")
+        deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
+        first_run = True
+
+        if self.force:
+            if os.path.isdir(deps_dir):
+                shutil.rmtree(deps_dir)
+
+        if not os.path.isdir(deps_dir):
+            os.makedirs(deps_dir)
+
+        # Read file with installed dependencies, if exist
+        installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt")
+        if os.path.exists(installed_deps_file):
+            installed_deps = [l.strip() for l in open(installed_deps_file)]
+        else:
+            installed_deps = []
+
+        # list of dependencies that need to be updated
+        update_deps = list(set(packages) - set(installed_deps))
+
+        for dep in packages:
+            dep_name = dep.split("-")[0]
+
+            # Don't download CMake if already exists in PATH
+            if dep_name == "cmake":
+                if spawn.find_executable(dep_name):
+                    continue
+
+            dep_dir = os.path.join(deps_dir, dep_name)
+            # if not installed or need to be updated
+            if not os.path.exists(dep_dir) or dep in update_deps:
+                if first_run:
+                    print "Installing missing MSVC dependencies..."
+                    first_run = False
+
+                dep_version_dir = os.path.join(deps_dir, dep)
+
+                if os.path.exists(dep_version_dir):
+                    shutil.rmtree(dep_version_dir)
+
+                dep_zip = dep_version_dir + ".zip"
+                if not os.path.isfile(dep_zip):
+                    download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip)
+
+                print "Extracting %s..." % dep,
+                extract(dep_zip, deps_dir)
+                print "done"
+
+                # Delete directory if exist
+                if os.path.exists(dep_dir):
+                    shutil.rmtree(dep_dir)
+                os.rename(dep_version_dir, dep_dir)
+
+        # Write in installed-dependencies.txt file
+        with open(installed_deps_file, 'w') as installed_file:
+            for line in packages:
+                installed_file.write(line + "\n")
+
+    def install_mobile_android_packages(self):
+        sys.exit('We do not support building Android on Windows. Sorry!')

python/servo/build_commands.py

@@ -257,16 +257,14 @@ class MachCommands(CommandBase):
                 # On windows, copy in our manifest
                 shutil.copy(path.join(self.get_top_dir(), "components", "servo", "servo.exe.manifest"),
                             servo_exe_dir)
-                if "msvc" in host_triple():
+                if "msvc" in (target or host_triple()):
+                    msvc_x64 = "64" if "x86_64" in (target or host_triple()) else ""
                     # on msvc builds, use editbin to change the subsystem to windows
                     call(["editbin", "/nologo", "/subsystem:windows", path.join(servo_exe_dir, "servo.exe")],
                          verbose=verbose)
                     # on msvc, we need to copy in some DLLs in to the servo.exe dir
                     for ssl_lib in ["ssleay32md.dll", "libeay32md.dll"]:
-                        shutil.copy(path.join(os.getenv('OPENSSL_LIB_DIR'), "../bin64", ssl_lib),
-                                    servo_exe_dir)
-                    for ffmpeg_lib in ["avutil-55.dll", "avformat-57.dll", "avcodec-57.dll", "swresample-2.dll"]:
-                        shutil.copy(path.join(os.getenv('FFMPEG_LIB_DIR'), "../bin", ffmpeg_lib),
+                        shutil.copy(path.join(env['OPENSSL_LIB_DIR'], "../bin" + msvc_x64, ssl_lib),
                                     servo_exe_dir)
 
                 elif sys.platform == "darwin":

python/servo/command_base.py

@@ -372,6 +372,18 @@ class CommandBase(object):
             env['PATH'] = env['PATH'].encode('ascii', 'ignore')
         extra_path = []
         extra_lib = []
+        if "msvc" in (target or host_triple()):
+            msvc_x64 = "64" if "x86_64" in (target or host_triple()) else ""
+            msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies")
+            extra_path += [path.join(msvc_deps_dir, "cmake", "bin")]
+            extra_path += [path.join(msvc_deps_dir, "ninja", "bin")]
+            # Link openssl
+            env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include")
+            env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64)
+            env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD"
+            # Link moztools
+            env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin")
+
         if not self.config["tools"]["system-rust"] \
                 or self.config["tools"]["rust-root"]:
             env["RUST_ROOT"] = self.config["tools"]["rust-root"]
@@ -493,6 +505,8 @@ class CommandBase(object):
         if self.context.bootstrapped:
             return
 
+        target_platform = target or host_triple()
+
         rust_root = self.config["tools"]["rust-root"]
         rustc_path = path.join(
             rust_root, "rustc", "bin", "rustc" + BIN_SUFFIX
@@ -501,9 +515,13 @@ class CommandBase(object):
 
         base_target_path = path.join(rust_root, "rustc", "lib", "rustlib")
 
-        target_path = path.join(base_target_path, target or host_triple())
+        target_path = path.join(base_target_path, target_platform)
         target_exists = path.exists(target_path)
 
+        # Always check if all needed MSVC dependencies are installed
+        if "msvc" in target_platform:
+            Registrar.dispatch("bootstrap", context=self.context)
+
         if not (self.config['tools']['system-rust'] or (rustc_binary_exists and target_exists)):
             print("looking for rustc at %s" % (rustc_path))
             Registrar.dispatch("bootstrap-rust", context=self.context, target=filter(None, [target]),