mirror of
https://github.com/servo/servo.git
synced 2025-06-06 16:45:39 +00:00
Clean up and simplify existing mach bootstrap
- Default to interactive mode and remove the `--interactive` flag - Use `--force` to skip interactivity - Change MSVC dependency storage organization on disk: put each version into its own folder and directly refer to the versioned folders, providing immutability and making the installation list redundant - Reuse `host_triple()` function to fix broken bootstrapper dispatching - Simplify code: - Remove or inline many unused and redudant functions and variables - Prefer plain functions to classes - Consolidate into fewer files, remove unnecessary bootstrapper/ dir - Improve Python style - Sort dependency list
This commit is contained in:
Aneesh Agrawal
parent
ef900cbdcb
commit
60a1503b29
9 changed files with 137 additions and 281 deletions
110
python/servo/bootstrap.py
Normal file
110
python/servo/bootstrap.py
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
from distutils.spawn import find_executable
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import servo.packages as packages
|
||||
from servo.util import extract, download_file, host_triple
|
||||
|
||||
|
||||
def windows_gnu(context, force=False):
|
||||
'''Bootstrapper for msys2 based environments for building in Windows.'''
|
||||
|
||||
if not find_executable('pacman'):
|
||||
print(
|
||||
'The Windows GNU bootstrapper only works with msys2 with pacman. '
|
||||
'Get msys2 at http://msys2.github.io/'
|
||||
)
|
||||
return 1
|
||||
|
||||
# Ensure repositories are up to date
|
||||
command = ['pacman', '--sync', '--refresh']
|
||||
subprocess.check_call(command)
|
||||
|
||||
# Install packages
|
||||
command = ['pacman', '--sync', '--needed']
|
||||
if force:
|
||||
command.append('--noconfirm')
|
||||
subprocess.check_call(command + list(packages.WINDOWS_GNU))
|
||||
|
||||
# Downgrade GCC to 5.4.0-1
|
||||
gcc_pkgs = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
|
||||
gcc_version = "5.4.0-1"
|
||||
mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
|
||||
gcc_list = [mingw_url.format(gcc, gcc_version) for gcc in gcc_pkgs]
|
||||
|
||||
# Note: `--upgrade` also does downgrades
|
||||
downgrade_command = ['pacman', '--upgrade']
|
||||
if force:
|
||||
downgrade_command.append('--noconfirm')
|
||||
subprocess.check_call(downgrade_command + gcc_list)
|
||||
|
||||
|
||||
def windows_msvc(context, force=False):
|
||||
'''Bootstrapper for MSVC building on Windows.'''
|
||||
|
||||
deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
|
||||
deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
|
||||
|
||||
def version(package):
|
||||
return packages.WINDOWS_MSVC[package]
|
||||
|
||||
def package_dir(package):
|
||||
return os.path.join(deps_dir, package, version(package))
|
||||
|
||||
to_install = {}
|
||||
for package in packages.WINDOWS_MSVC:
|
||||
# Don't install CMake if it already exists in PATH
|
||||
if package == "cmake" and find_executable(package):
|
||||
continue
|
||||
|
||||
if not os.path.isdir(package_dir(package)):
|
||||
to_install[package] = version(package)
|
||||
|
||||
if not to_install:
|
||||
return 0
|
||||
|
||||
print("Installing missing MSVC dependencies...")
|
||||
for package in to_install:
|
||||
full_spec = '{}-{}'.format(package, version(package))
|
||||
|
||||
parent_dir = os.path.dirname(package_dir(package))
|
||||
if not os.path.isdir(parent_dir):
|
||||
os.makedirs(parent_dir)
|
||||
|
||||
zip_path = package_dir(package) + ".zip"
|
||||
if not os.path.isfile(zip_path):
|
||||
zip_url = "{}{}.zip".format(deps_url, full_spec)
|
||||
download_file(full_spec, zip_url, zip_path)
|
||||
|
||||
print("Extracting {}...".format(full_spec), end='')
|
||||
extract(zip_path, deps_dir)
|
||||
print("done")
|
||||
|
||||
extracted_path = os.path.join(deps_dir, full_spec)
|
||||
os.rename(extracted_path, package_dir(package))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def bootstrap(context, force=False):
|
||||
'''Dispatches to the right bootstrapping function for the OS.'''
|
||||
|
||||
bootstrapper = None
|
||||
|
||||
if "windows-gnu" in host_triple():
|
||||
bootstrapper = windows_gnu
|
||||
elif "windows-msvc" in host_triple():
|
||||
bootstrapper = windows_msvc
|
||||
|
||||
if bootstrapper is None:
|
||||
print('Bootstrap support is not yet available for your OS.')
|
||||
return 1
|
||||
|
||||
return bootstrapper(context, force=force)
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
# option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import base64
|
||||
import json
|
||||
|
|
@ -24,6 +24,7 @@ from mach.decorators import (
|
|||
Command,
|
||||
)
|
||||
|
||||
import servo.bootstrap as bootstrap
|
||||
from servo.command_base import CommandBase, BIN_SUFFIX
|
||||
from servo.util import download_bytes, download_file, extract, host_triple
|
||||
|
||||
|
|
@ -44,17 +45,11 @@ class MachCommands(CommandBase):
|
|||
@Command('bootstrap',
|
||||
description='Install required packages for building.',
|
||||
category='bootstrap')
|
||||
@CommandArgument('--interactive', "-i",
|
||||
action='store_true',
|
||||
help='Need to answer any (Y/n) interactive prompts.')
|
||||
@CommandArgument('--force', '-f',
|
||||
action='store_true',
|
||||
help='Force reinstall packages')
|
||||
def bootstrap(self, interactive=False, force=False):
|
||||
from servo.bootstrapper.bootstrap import Bootstrapper
|
||||
|
||||
bootstrapper = Bootstrapper(self.context)
|
||||
bootstrapper.bootstrap(interactive=interactive, force=force)
|
||||
help='Boostrap without confirmation')
|
||||
def bootstrap(self, force=False):
|
||||
return bootstrap.bootstrap(self.context, force=force)
|
||||
|
||||
@Command('bootstrap-rust',
|
||||
description='Download the Rust compiler',
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import distutils
|
||||
import subprocess
|
||||
|
||||
|
||||
class BaseBootstrapper(object):
|
||||
"""Base class for system bootstrappers."""
|
||||
|
||||
def __init__(self, interactive=False):
|
||||
self.package_manager_updated = False
|
||||
self.interactive = interactive
|
||||
|
||||
def ensure_system_packages(self):
|
||||
'''
|
||||
Check for missing packages.
|
||||
'''
|
||||
raise NotImplementedError('%s must implement ensure_system_packages()' %
|
||||
__name__)
|
||||
|
||||
def install_system_packages(self):
|
||||
'''
|
||||
Install packages required to build Servo.
|
||||
'''
|
||||
raise NotImplementedError('%s must implement install_system_packages()' %
|
||||
__name__)
|
||||
|
||||
def which(self, name):
|
||||
"""Python implementation of which.
|
||||
|
||||
It returns the path of an executable or None if it couldn't be found.
|
||||
"""
|
||||
return distutils.spawn.find_executable(name)
|
||||
|
||||
def check_output(self, *args, **kwargs):
|
||||
"""Run subprocess.check_output."""
|
||||
return subprocess.check_output(*args, **kwargs)
|
||||
|
||||
def _ensure_package_manager_updated(self):
|
||||
if self.package_manager_updated:
|
||||
return
|
||||
|
||||
self._update_package_manager()
|
||||
self.package_manager_updated = True
|
||||
|
||||
def _update_package_manager(self):
|
||||
"""Updates the package manager's manifests/package list.
|
||||
|
||||
This should be defined in child classes.
|
||||
"""
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
|
||||
from windows_gnu import WindowsGnuBootstrapper
|
||||
from windows_msvc import WindowsMsvcBootstrapper
|
||||
|
||||
|
||||
class Bootstrapper(object):
|
||||
"""Main class that performs system bootstrap."""
|
||||
|
||||
def __init__(self, context):
|
||||
self.instance = None
|
||||
cls = None
|
||||
args = {}
|
||||
|
||||
if sys.platform.startswith('msys'):
|
||||
cls = WindowsGnuBootstrapper
|
||||
|
||||
elif sys.platform.startswith('win32'):
|
||||
cls = WindowsMsvcBootstrapper
|
||||
|
||||
if cls is None:
|
||||
sys.exit('Bootstrap support is not yet available for your OS.')
|
||||
|
||||
self.instance = cls(**args)
|
||||
self.instance.context = context
|
||||
|
||||
def bootstrap(self, interactive=False, force=False):
|
||||
self.instance.interactive = interactive
|
||||
self.instance.force = force
|
||||
|
||||
if force:
|
||||
self.instance.install_system_packages()
|
||||
else:
|
||||
self.instance.ensure_system_packages()
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
from base import BaseBootstrapper
|
||||
from packages import WINDOWS_GNU as deps
|
||||
|
||||
|
||||
class WindowsGnuBootstrapper(BaseBootstrapper):
|
||||
'''Bootstrapper for msys2 based environments for building in Windows.'''
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
BaseBootstrapper.__init__(self, **kwargs)
|
||||
|
||||
if not self.which('pacman'):
|
||||
raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at '
|
||||
'http://msys2.github.io/')
|
||||
|
||||
def ensure_system_packages(self):
|
||||
install_packages = []
|
||||
for p in deps:
|
||||
command = ['pacman', '-Qs', p]
|
||||
if self.run_check(command):
|
||||
install_packages += [p]
|
||||
if install_packages:
|
||||
install_packages(install_packages)
|
||||
|
||||
def install_system_packages(self, packages=deps):
|
||||
self._ensure_package_manager_updated()
|
||||
self.pacman_install(*packages)
|
||||
|
||||
def _update_package_manager(self):
|
||||
self.pacman_update()
|
||||
|
||||
def run(self, command):
|
||||
subprocess.check_call(command, stdin=sys.stdin)
|
||||
|
||||
def run_check(self, command):
|
||||
return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def pacman_update(self):
|
||||
command = ['pacman', '--sync', '--refresh']
|
||||
self.run(command)
|
||||
|
||||
def pacman_upgrade(self):
|
||||
command = ['pacman', '--sync', '--refresh', '--sysupgrade']
|
||||
self.run(command)
|
||||
|
||||
def pacman_install(self, *packages):
|
||||
command = ['pacman', '--sync']
|
||||
if not self.force:
|
||||
command.append('--needed')
|
||||
if not self.interactive:
|
||||
command.append('--noconfirm')
|
||||
command.extend(packages)
|
||||
self.run(command)
|
||||
|
||||
# downgrade GCC to 5.4.0-1
|
||||
gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
|
||||
gcc_version = "5.4.0-1"
|
||||
mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
|
||||
gcc_list = []
|
||||
for gcc in gcc_type:
|
||||
gcc_list += [mingw_url.format(gcc, gcc_version)]
|
||||
downgrade_command = ['pacman', '-U']
|
||||
if not self.interactive:
|
||||
downgrade_command.append('--noconfirm')
|
||||
downgrade_command.extend(gcc_list)
|
||||
self.run(downgrade_command)
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from distutils import spawn
|
||||
|
||||
from servo.util import extract, download_file
|
||||
from base import BaseBootstrapper
|
||||
from packages import WINDOWS_MSVC as deps
|
||||
|
||||
|
||||
class WindowsMsvcBootstrapper(BaseBootstrapper):
|
||||
'''Bootstrapper for MSVC building on Windows.'''
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
BaseBootstrapper.__init__(self, **kwargs)
|
||||
|
||||
def ensure_system_packages(self):
|
||||
self.install_system_packages()
|
||||
|
||||
def install_system_packages(self, packages=deps):
|
||||
|
||||
deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies")
|
||||
deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
|
||||
first_run = True
|
||||
|
||||
if self.force:
|
||||
if os.path.isdir(deps_dir):
|
||||
shutil.rmtree(deps_dir)
|
||||
|
||||
if not os.path.isdir(deps_dir):
|
||||
os.makedirs(deps_dir)
|
||||
|
||||
# Read file with installed dependencies, if exist
|
||||
installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt")
|
||||
if os.path.exists(installed_deps_file):
|
||||
installed_deps = [l.strip() for l in open(installed_deps_file)]
|
||||
else:
|
||||
installed_deps = []
|
||||
|
||||
# list of dependencies that need to be updated
|
||||
update_deps = list(set(packages) - set(installed_deps))
|
||||
|
||||
for dep in packages:
|
||||
dep_name = dep.split("-")[0]
|
||||
|
||||
# Don't download CMake if already exists in PATH
|
||||
if dep_name == "cmake":
|
||||
if spawn.find_executable(dep_name):
|
||||
continue
|
||||
|
||||
dep_dir = os.path.join(deps_dir, dep_name)
|
||||
# if not installed or need to be updated
|
||||
if not os.path.exists(dep_dir) or dep in update_deps:
|
||||
if first_run:
|
||||
print "Installing missing MSVC dependencies..."
|
||||
first_run = False
|
||||
|
||||
dep_version_dir = os.path.join(deps_dir, dep)
|
||||
|
||||
if os.path.exists(dep_version_dir):
|
||||
shutil.rmtree(dep_version_dir)
|
||||
|
||||
dep_zip = dep_version_dir + ".zip"
|
||||
if not os.path.isfile(dep_zip):
|
||||
download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip)
|
||||
|
||||
print "Extracting %s..." % dep,
|
||||
extract(dep_zip, deps_dir)
|
||||
print "done"
|
||||
|
||||
# Delete directory if exist
|
||||
if os.path.exists(dep_dir):
|
||||
shutil.rmtree(dep_dir)
|
||||
os.rename(dep_version_dir, dep_dir)
|
||||
|
||||
# Write in installed-dependencies.txt file
|
||||
with open(installed_deps_file, 'w') as installed_file:
|
||||
for line in packages:
|
||||
installed_file.write(line + "\n")
|
||||
|
|
@ -19,12 +19,12 @@ from subprocess import PIPE
|
|||
import sys
|
||||
import tarfile
|
||||
|
||||
from mach.registrar import Registrar
|
||||
import toml
|
||||
|
||||
from mach.registrar import Registrar
|
||||
from servo.packages import WINDOWS_MSVC as msvc_deps
|
||||
from servo.util import host_triple
|
||||
|
||||
|
||||
BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
|
||||
|
||||
|
||||
|
|
@ -384,14 +384,18 @@ class CommandBase(object):
|
|||
if "msvc" in (target or host_triple()):
|
||||
msvc_x64 = "64" if "x86_64" in (target or host_triple()) else ""
|
||||
msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies")
|
||||
extra_path += [path.join(msvc_deps_dir, "cmake", "bin")]
|
||||
extra_path += [path.join(msvc_deps_dir, "ninja", "bin")]
|
||||
|
||||
def package_dir(package):
|
||||
return path.join(msvc_deps_dir, package, msvc_deps[package])
|
||||
|
||||
extra_path += [path.join(package_dir("cmake"), "bin")]
|
||||
extra_path += [path.join(package_dir("ninja"), "bin")]
|
||||
# Link openssl
|
||||
env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include")
|
||||
env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64)
|
||||
env["OPENSSL_INCLUDE_DIR"] = path.join(package_dir("openssl"), "include")
|
||||
env["OPENSSL_LIB_DIR"] = path.join(package_dir("openssl"), "lib" + msvc_x64)
|
||||
env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD"
|
||||
# Link moztools
|
||||
env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin")
|
||||
env["MOZTOOLS_PATH"] = path.join(package_dir("moztools"), "bin")
|
||||
|
||||
if is_windows():
|
||||
if not os.environ.get("NATIVE_WIN32_PYTHON"):
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# Listed all packages for different platforms in one file
|
||||
|
||||
WINDOWS_GNU = [
|
||||
WINDOWS_GNU = set([
|
||||
"diffutils",
|
||||
"make",
|
||||
"mingw-w64-x86_64-toolchain",
|
||||
"mingw-w64-x86_64-freetype",
|
||||
"mingw-w64-x86_64-icu",
|
||||
|
|
@ -12,17 +12,15 @@ WINDOWS_GNU = [
|
|||
"mingw-w64-x86_64-ca-certificates",
|
||||
"mingw-w64-x86_64-expat",
|
||||
"mingw-w64-x86_64-cmake",
|
||||
"tar",
|
||||
"diffutils",
|
||||
"patch",
|
||||
"patchutils",
|
||||
"make",
|
||||
"python2-setuptools",
|
||||
]
|
||||
"tar",
|
||||
])
|
||||
|
||||
WINDOWS_MSVC = [
|
||||
"cmake-3.6.1",
|
||||
"ninja-1.7.1",
|
||||
"openssl-1.0.1t-vs2015",
|
||||
"moztools-0.0.1-5",
|
||||
]
|
||||
WINDOWS_MSVC = {
|
||||
"cmake": "3.6.1",
|
||||
"moztools": "0.0.1-5",
|
||||
"ninja": "1.7.1",
|
||||
"openssl": "1.0.1t-vs2015",
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue