mirror of
https://github.com/servo/servo.git
synced 2025-06-10 09:33:13 +00:00
Auto merge of #14974 - aneeshusa:add-mach-bootstrap, r=metajack
Use Salt for mach bootstrap <!-- Please describe your changes on the following line: --> This is currently WIP, but allows Salt for `mach bootstrap`. Not looking for review yet, just posting for visibility. You can run `./mach bootstrap` and Salt will run, letting you know what changes it would make for bootstrapping (doesn't actually run yet though). Currently, this reads from saltfs using gitfs, meaning it always tracks the master branch. (Note that this is blocked on https://github.com/servo/saltfs/pull/577 landing; in the meantime, I've included a small workaround in this PR to pull from my updated saltfs branch, which will need to be removed.) In the future, the relevant Salt code may be merged in tree as part of Docker support for TC, and the bootstrapper should be updated to read from in tree. Also, our Windows machines have not been Salted, so the existing Windows bootstrappers are retained. TODO: - [x] Hook into existing bootstrapping code less hackily - [x] Actually bootstrap instead of always using `test=True` mode (includes sudo) - [x] Default to interactive mode (test first, then ask), with a force flag - [x] Don't require running from the repository root directory - [x] Make it easy to add support for other distros --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: --> - [ ] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [ ] These changes fix #__ (github issue number if applicable). <!-- Either: --> - [ ] There are tests for these changes OR - [x] These changes do not require tests because they should be verified manually <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. --> <!-- Reviewable:start --> --- This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/14974) <!-- Reviewable:end -->
This commit is contained in:
bors-servo
GitHub
commit
1f76aa6ef7
15 changed files with 431 additions and 443 deletions
|
|
@ -118,7 +118,11 @@ def _activate_virtualenv(topdir):
|
|||
if not virtualenv:
|
||||
sys.exit("Python virtualenv is not installed. Please install it prior to running mach.")
|
||||
|
||||
process = Popen([virtualenv, "-p", python, virtualenv_path], stdout=PIPE, stderr=PIPE)
|
||||
process = Popen(
|
||||
[virtualenv, "-p", python, "--system-site-packages", virtualenv_path],
|
||||
stdout=PIPE,
|
||||
stderr=PIPE
|
||||
)
|
||||
process.wait()
|
||||
if process.returncode:
|
||||
out, err = process.communicate()
|
||||
|
|
@ -153,7 +157,7 @@ def _activate_virtualenv(topdir):
|
|||
if not pip:
|
||||
sys.exit("Python pip is either not installed or not found in virtualenv.")
|
||||
|
||||
process = Popen([pip, "install", "-q", "-U", "pip"], stdout=PIPE, stderr=PIPE)
|
||||
process = Popen([pip, "install", "-q", "-I", "-U", "pip"], stdout=PIPE, stderr=PIPE)
|
||||
process.wait()
|
||||
if process.returncode:
|
||||
out, err = process.communicate()
|
||||
|
|
@ -175,7 +179,7 @@ def _activate_virtualenv(topdir):
|
|||
if not pip:
|
||||
sys.exit("Python pip is either not installed or not found in virtualenv.")
|
||||
|
||||
process = Popen([pip, "install", "-q", "-r", req_path], stdout=PIPE, stderr=PIPE)
|
||||
process = Popen([pip, "install", "-q", "-I", "-r", req_path], stdout=PIPE, stderr=PIPE)
|
||||
process.wait()
|
||||
if process.returncode:
|
||||
out, err = process.communicate()
|
||||
|
|
|
|||
6
python/requirements-salt.txt
Normal file
6
python/requirements-salt.txt
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
# Ensure all versions are pinned for repeatability,
|
||||
# since `--system-site-packages` is enabled
|
||||
|
||||
# For boostrapping, make sure versions match those in saltfs
|
||||
salt == 2016.3.4
|
||||
GitPython == 0.3.2
|
||||
|
|
@ -1,3 +1,6 @@
|
|||
# Ensure all versions are pinned for repeatability,
|
||||
# since `--system-site-packages` is enabled
|
||||
|
||||
blessings == 1.6
|
||||
mach == 0.6.0
|
||||
mozdebug == 0.1
|
||||
|
|
|
|||
229
python/servo/bootstrap.py
Normal file
229
python/servo/bootstrap.py
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
from distutils.spawn import find_executable
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import servo.packages as packages
|
||||
from servo.util import extract, download_file, host_triple
|
||||
|
||||
|
||||
def salt(context, force=False):
|
||||
# Ensure Salt is installed in the virtualenv
|
||||
# It's not instaled globally because it's a large, non-required dependency,
|
||||
# and the installation fails on Windows
|
||||
print("Checking Salt installation...", end='')
|
||||
reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt')
|
||||
process = subprocess.Popen(
|
||||
["pip", "install", "-q", "-I", "-r", reqs_path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
process.wait()
|
||||
if process.returncode:
|
||||
out, err = process.communicate()
|
||||
print('failed to install Salt via pip:')
|
||||
print('Output: {}\nError: {}'.format(out, err))
|
||||
return 1
|
||||
print("done")
|
||||
|
||||
salt_root = os.path.join(context.sharedir, 'salt')
|
||||
config_dir = os.path.join(salt_root, 'etc', 'salt')
|
||||
pillar_dir = os.path.join(config_dir, 'pillars')
|
||||
|
||||
# In order to allow `mach bootstrap` to work from any CWD,
|
||||
# the `root_dir` must be an absolute path.
|
||||
# We place it under `context.sharedir` because
|
||||
# Salt caches data (e.g. gitfs files) in its `var` subdirectory.
|
||||
# Hence, dynamically generate the config with an appropriate `root_dir`
|
||||
# and serialize it as JSON (which is valid YAML).
|
||||
config = {
|
||||
'fileserver_backend': ['git'],
|
||||
'gitfs_env_whitelist': 'base',
|
||||
'gitfs_provider': 'gitpython',
|
||||
'gitfs_remotes': [
|
||||
'https://github.com/servo/saltfs.git',
|
||||
],
|
||||
'hash_type': 'sha384',
|
||||
'master': 'localhost',
|
||||
'root_dir': salt_root,
|
||||
'state_output': 'changes',
|
||||
'state_tabular': True,
|
||||
}
|
||||
|
||||
if not os.path.exists(config_dir):
|
||||
os.makedirs(config_dir, mode=0o700)
|
||||
with open(os.path.join(config_dir, 'minion'), 'w') as config_file:
|
||||
config_file.write(json.dumps(config) + '\n')
|
||||
|
||||
# Similarly, the pillar data is created dynamically
|
||||
# and temporarily serialized to disk.
|
||||
# This dynamism is not yet used, but will be in the future
|
||||
# to enable Android bootstrapping by using
|
||||
# context.sharedir as a location for Android packages.
|
||||
pillar = {
|
||||
'top.sls': {
|
||||
'base': {
|
||||
'*': ['bootstrap'],
|
||||
},
|
||||
},
|
||||
'bootstrap.sls': {
|
||||
'fully_managed': False,
|
||||
},
|
||||
}
|
||||
if os.path.exists(pillar_dir):
|
||||
shutil.rmtree(pillar_dir)
|
||||
os.makedirs(pillar_dir, mode=0o700)
|
||||
for filename in pillar:
|
||||
with open(os.path.join(pillar_dir, filename), 'w') as pillar_file:
|
||||
pillar_file.write(json.dumps(pillar[filename]) + '\n')
|
||||
|
||||
cmd = [
|
||||
'sudo',
|
||||
# sudo escapes from the venv, need to use full path
|
||||
find_executable('salt-call'),
|
||||
'--local',
|
||||
'--config-dir={}'.format(config_dir),
|
||||
'--pillar-root={}'.format(pillar_dir),
|
||||
'state.apply',
|
||||
'servo-build-dependencies',
|
||||
]
|
||||
|
||||
if not force:
|
||||
print('Running bootstrap in dry-run mode to show changes')
|
||||
# Because `test=True` mode runs each state individually without
|
||||
# considering how required/previous states affect the system,
|
||||
# it will often report states with requisites as failing due
|
||||
# to the requisites not actually being run,
|
||||
# even though these are spurious and will succeed during
|
||||
# the actual highstate.
|
||||
# Hence `--retcode-passthrough` is not helpful in dry-run mode,
|
||||
# so only detect failures of the actual salt-call binary itself.
|
||||
retcode = subprocess.call(cmd + ['test=True'])
|
||||
if retcode != 0:
|
||||
print('Something went wrong while bootstrapping')
|
||||
return retcode
|
||||
|
||||
proceed = raw_input(
|
||||
'Proposed changes are above, proceed with bootstrap? [y/N]: '
|
||||
)
|
||||
if proceed.lower() not in ['y', 'yes']:
|
||||
return 0
|
||||
|
||||
print('')
|
||||
|
||||
print('Running Salt bootstrap')
|
||||
retcode = subprocess.call(cmd + ['--retcode-passthrough'])
|
||||
if retcode == 0:
|
||||
print('Salt bootstrapping complete')
|
||||
else:
|
||||
print('Salt bootstrapping encountered errors')
|
||||
return retcode
|
||||
|
||||
|
||||
def windows_gnu(context, force=False):
|
||||
'''Bootstrapper for msys2 based environments for building in Windows.'''
|
||||
|
||||
if not find_executable('pacman'):
|
||||
print(
|
||||
'The Windows GNU bootstrapper only works with msys2 with pacman. '
|
||||
'Get msys2 at http://msys2.github.io/'
|
||||
)
|
||||
return 1
|
||||
|
||||
# Ensure repositories are up to date
|
||||
command = ['pacman', '--sync', '--refresh']
|
||||
subprocess.check_call(command)
|
||||
|
||||
# Install packages
|
||||
command = ['pacman', '--sync', '--needed']
|
||||
if force:
|
||||
command.append('--noconfirm')
|
||||
subprocess.check_call(command + list(packages.WINDOWS_GNU))
|
||||
|
||||
# Downgrade GCC to 5.4.0-1
|
||||
gcc_pkgs = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
|
||||
gcc_version = "5.4.0-1"
|
||||
mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
|
||||
gcc_list = [mingw_url.format(gcc, gcc_version) for gcc in gcc_pkgs]
|
||||
|
||||
# Note: `--upgrade` also does downgrades
|
||||
downgrade_command = ['pacman', '--upgrade']
|
||||
if force:
|
||||
downgrade_command.append('--noconfirm')
|
||||
subprocess.check_call(downgrade_command + gcc_list)
|
||||
|
||||
|
||||
def windows_msvc(context, force=False):
|
||||
'''Bootstrapper for MSVC building on Windows.'''
|
||||
|
||||
deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
|
||||
deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
|
||||
|
||||
def version(package):
|
||||
return packages.WINDOWS_MSVC[package]
|
||||
|
||||
def package_dir(package):
|
||||
return os.path.join(deps_dir, package, version(package))
|
||||
|
||||
to_install = {}
|
||||
for package in packages.WINDOWS_MSVC:
|
||||
# Don't install CMake if it already exists in PATH
|
||||
if package == "cmake" and find_executable(package):
|
||||
continue
|
||||
|
||||
if not os.path.isdir(package_dir(package)):
|
||||
to_install[package] = version(package)
|
||||
|
||||
if not to_install:
|
||||
return 0
|
||||
|
||||
print("Installing missing MSVC dependencies...")
|
||||
for package in to_install:
|
||||
full_spec = '{}-{}'.format(package, version(package))
|
||||
|
||||
parent_dir = os.path.dirname(package_dir(package))
|
||||
if not os.path.isdir(parent_dir):
|
||||
os.makedirs(parent_dir)
|
||||
|
||||
zip_path = package_dir(package) + ".zip"
|
||||
if not os.path.isfile(zip_path):
|
||||
zip_url = "{}{}.zip".format(deps_url, full_spec)
|
||||
download_file(full_spec, zip_url, zip_path)
|
||||
|
||||
print("Extracting {}...".format(full_spec), end='')
|
||||
extract(zip_path, deps_dir)
|
||||
print("done")
|
||||
|
||||
extracted_path = os.path.join(deps_dir, full_spec)
|
||||
os.rename(extracted_path, package_dir(package))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def bootstrap(context, force=False):
|
||||
'''Dispatches to the right bootstrapping function for the OS.'''
|
||||
|
||||
bootstrapper = None
|
||||
|
||||
if "windows-gnu" in host_triple():
|
||||
bootstrapper = windows_gnu
|
||||
elif "windows-msvc" in host_triple():
|
||||
bootstrapper = windows_msvc
|
||||
elif "linux-gnu" in host_triple():
|
||||
distro, version, _ = platform.linux_distribution()
|
||||
if distro == 'Ubuntu' and version == '14.04':
|
||||
bootstrapper = salt
|
||||
|
||||
if bootstrapper is None:
|
||||
print('Bootstrap support is not yet available for your OS.')
|
||||
return 1
|
||||
|
||||
return bootstrapper(context, force=force)
|
||||
|
|
@ -7,8 +7,7 @@
|
|||
# option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
from socket import error as socket_error
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import base64
|
||||
import json
|
||||
|
|
@ -17,9 +16,6 @@ import os.path as path
|
|||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import StringIO
|
||||
import tarfile
|
||||
import zipfile
|
||||
import urllib2
|
||||
|
||||
from mach.decorators import (
|
||||
|
|
@ -28,93 +24,9 @@ from mach.decorators import (
|
|||
Command,
|
||||
)
|
||||
|
||||
from servo.command_base import CommandBase, host_triple, BIN_SUFFIX
|
||||
|
||||
|
||||
def download(desc, src, writer, start_byte=0):
|
||||
if start_byte:
|
||||
print("Resuming download of %s..." % desc)
|
||||
else:
|
||||
print("Downloading %s..." % desc)
|
||||
dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
|
||||
|
||||
try:
|
||||
req = urllib2.Request(src)
|
||||
if start_byte:
|
||||
req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
|
||||
resp = urllib2.urlopen(req)
|
||||
|
||||
fsize = None
|
||||
if resp.info().getheader('Content-Length'):
|
||||
fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte
|
||||
|
||||
recved = start_byte
|
||||
chunk_size = 8192
|
||||
|
||||
while True:
|
||||
chunk = resp.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
recved += len(chunk)
|
||||
if not dumb:
|
||||
if fsize is not None:
|
||||
pct = recved * 100.0 / fsize
|
||||
print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
|
||||
|
||||
sys.stdout.flush()
|
||||
writer.write(chunk)
|
||||
|
||||
if not dumb:
|
||||
print()
|
||||
except urllib2.HTTPError, e:
|
||||
print("Download failed (%d): %s - %s" % (e.code, e.reason, src))
|
||||
if e.code == 403:
|
||||
print("No Rust compiler binary available for this platform. "
|
||||
"Please see https://github.com/servo/servo/#prerequisites")
|
||||
sys.exit(1)
|
||||
except urllib2.URLError, e:
|
||||
print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src))
|
||||
sys.exit(1)
|
||||
except socket_error, e:
|
||||
print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e))
|
||||
sys.exit(1)
|
||||
except KeyboardInterrupt:
|
||||
writer.flush()
|
||||
raise
|
||||
|
||||
|
||||
def download_file(desc, src, dst):
|
||||
tmp_path = dst + ".part"
|
||||
try:
|
||||
start_byte = os.path.getsize(tmp_path)
|
||||
with open(tmp_path, 'ab') as fd:
|
||||
download(desc, src, fd, start_byte=start_byte)
|
||||
except os.error:
|
||||
with open(tmp_path, 'wb') as fd:
|
||||
download(desc, src, fd)
|
||||
os.rename(tmp_path, dst)
|
||||
|
||||
|
||||
def download_bytes(desc, src):
|
||||
content_writer = StringIO.StringIO()
|
||||
download(desc, src, content_writer)
|
||||
return content_writer.getvalue()
|
||||
|
||||
|
||||
def extract(src, dst, movedir=None):
|
||||
if src.endswith(".zip"):
|
||||
zipfile.ZipFile(src).extractall(dst)
|
||||
else:
|
||||
tarfile.open(src).extractall(dst)
|
||||
|
||||
if movedir:
|
||||
for f in os.listdir(movedir):
|
||||
frm = path.join(movedir, f)
|
||||
to = path.join(dst, f)
|
||||
os.rename(frm, to)
|
||||
os.rmdir(movedir)
|
||||
|
||||
os.remove(src)
|
||||
import servo.bootstrap as bootstrap
|
||||
from servo.command_base import CommandBase, BIN_SUFFIX
|
||||
from servo.util import download_bytes, download_file, extract, host_triple
|
||||
|
||||
|
||||
@CommandProvider
|
||||
|
|
@ -133,20 +45,11 @@ class MachCommands(CommandBase):
|
|||
@Command('bootstrap',
|
||||
description='Install required packages for building.',
|
||||
category='bootstrap')
|
||||
@CommandArgument('--interactive', "-i",
|
||||
action='store_true',
|
||||
help='Need to answer any (Y/n) interactive prompts.')
|
||||
@CommandArgument('--android',
|
||||
action='store_true',
|
||||
help='Install required packages for Android')
|
||||
@CommandArgument('--force', '-f',
|
||||
action='store_true',
|
||||
help='Force reinstall packages')
|
||||
def bootstrap(self, android=False, interactive=False, force=False):
|
||||
from servo.bootstrapper.bootstrap import Bootstrapper
|
||||
|
||||
bootstrapper = Bootstrapper(self.context)
|
||||
bootstrapper.bootstrap(android=android, interactive=interactive, force=force)
|
||||
help='Boostrap without confirmation')
|
||||
def bootstrap(self, force=False):
|
||||
return bootstrap.bootstrap(self.context, force=force)
|
||||
|
||||
@Command('bootstrap-rust',
|
||||
description='Download the Rust compiler',
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import distutils
|
||||
import subprocess
|
||||
|
||||
|
||||
class BaseBootstrapper(object):
|
||||
"""Base class for system bootstrappers."""
|
||||
|
||||
def __init__(self, interactive=False):
|
||||
self.package_manager_updated = False
|
||||
self.interactive = interactive
|
||||
|
||||
def ensure_system_packages(self):
|
||||
'''
|
||||
Check for missing packages.
|
||||
'''
|
||||
raise NotImplementedError('%s must implement ensure_system_packages()' %
|
||||
__name__)
|
||||
|
||||
def install_system_packages(self):
|
||||
'''
|
||||
Install packages required to build Servo.
|
||||
'''
|
||||
raise NotImplementedError('%s must implement install_system_packages()' %
|
||||
__name__)
|
||||
|
||||
def install_mobile_android_packages(self):
|
||||
'''
|
||||
Install packages required to build Servo for Android.
|
||||
'''
|
||||
raise NotImplementedError('Cannot bootstrap Servo for Android: '
|
||||
'%s does not yet implement install_mobile_android_packages()'
|
||||
% __name__)
|
||||
|
||||
def which(self, name):
|
||||
"""Python implementation of which.
|
||||
|
||||
It returns the path of an executable or None if it couldn't be found.
|
||||
"""
|
||||
return distutils.spawn.find_executable(name)
|
||||
|
||||
def check_output(self, *args, **kwargs):
|
||||
"""Run subprocess.check_output."""
|
||||
return subprocess.check_output(*args, **kwargs)
|
||||
|
||||
def _ensure_package_manager_updated(self):
|
||||
if self.package_manager_updated:
|
||||
return
|
||||
|
||||
self._update_package_manager()
|
||||
self.package_manager_updated = True
|
||||
|
||||
def _update_package_manager(self):
|
||||
"""Updates the package manager's manifests/package list.
|
||||
|
||||
This should be defined in child classes.
|
||||
"""
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
|
||||
from windows_gnu import WindowsGnuBootstrapper
|
||||
from windows_msvc import WindowsMsvcBootstrapper
|
||||
|
||||
|
||||
class Bootstrapper(object):
|
||||
"""Main class that performs system bootstrap."""
|
||||
|
||||
def __init__(self, context):
|
||||
self.instance = None
|
||||
cls = None
|
||||
args = {}
|
||||
|
||||
if sys.platform.startswith('msys'):
|
||||
cls = WindowsGnuBootstrapper
|
||||
|
||||
elif sys.platform.startswith('win32'):
|
||||
cls = WindowsMsvcBootstrapper
|
||||
|
||||
if cls is None:
|
||||
sys.exit('Bootstrap support is not yet available for your OS.')
|
||||
|
||||
self.instance = cls(**args)
|
||||
self.instance.context = context
|
||||
|
||||
def bootstrap(self, android=False, interactive=False, force=False):
|
||||
self.instance.interactive = interactive
|
||||
self.instance.force = force
|
||||
|
||||
if android:
|
||||
self.instance.install_mobile_android_packages()
|
||||
elif force:
|
||||
self.instance.install_system_packages()
|
||||
else:
|
||||
self.instance.ensure_system_packages()
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
from base import BaseBootstrapper
|
||||
from packages import WINDOWS_GNU as deps
|
||||
|
||||
|
||||
class WindowsGnuBootstrapper(BaseBootstrapper):
|
||||
'''Bootstrapper for msys2 based environments for building in Windows.'''
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
BaseBootstrapper.__init__(self, **kwargs)
|
||||
|
||||
if not self.which('pacman'):
|
||||
raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at '
|
||||
'http://msys2.github.io/')
|
||||
|
||||
def ensure_system_packages(self):
|
||||
install_packages = []
|
||||
for p in deps:
|
||||
command = ['pacman', '-Qs', p]
|
||||
if self.run_check(command):
|
||||
install_packages += [p]
|
||||
if install_packages:
|
||||
install_packages(install_packages)
|
||||
|
||||
def install_system_packages(self, packages=deps):
|
||||
self._ensure_package_manager_updated()
|
||||
self.pacman_install(*packages)
|
||||
|
||||
def install_mobile_android_packages(self):
|
||||
sys.exit('We do not support building Android on Windows. Sorry!')
|
||||
|
||||
def _update_package_manager(self):
|
||||
self.pacman_update()
|
||||
|
||||
def run(self, command):
|
||||
subprocess.check_call(command, stdin=sys.stdin)
|
||||
|
||||
def run_check(self, command):
|
||||
return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def pacman_update(self):
|
||||
command = ['pacman', '--sync', '--refresh']
|
||||
self.run(command)
|
||||
|
||||
def pacman_upgrade(self):
|
||||
command = ['pacman', '--sync', '--refresh', '--sysupgrade']
|
||||
self.run(command)
|
||||
|
||||
def pacman_install(self, *packages):
|
||||
command = ['pacman', '--sync']
|
||||
if not self.force:
|
||||
command.append('--needed')
|
||||
if not self.interactive:
|
||||
command.append('--noconfirm')
|
||||
command.extend(packages)
|
||||
self.run(command)
|
||||
|
||||
# downgrade GCC to 5.4.0-1
|
||||
gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
|
||||
gcc_version = "5.4.0-1"
|
||||
mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
|
||||
gcc_list = []
|
||||
for gcc in gcc_type:
|
||||
gcc_list += [mingw_url.format(gcc, gcc_version)]
|
||||
downgrade_command = ['pacman', '-U']
|
||||
if not self.interactive:
|
||||
downgrade_command.append('--noconfirm')
|
||||
downgrade_command.extend(gcc_list)
|
||||
self.run(downgrade_command)
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
from distutils import spawn
|
||||
|
||||
from base import BaseBootstrapper
|
||||
from packages import WINDOWS_MSVC as deps
|
||||
|
||||
|
||||
class WindowsMsvcBootstrapper(BaseBootstrapper):
|
||||
'''Bootstrapper for MSVC building on Windows.'''
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
BaseBootstrapper.__init__(self, **kwargs)
|
||||
|
||||
def ensure_system_packages(self):
|
||||
self.install_system_packages()
|
||||
|
||||
def install_system_packages(self, packages=deps):
|
||||
from servo.bootstrap_commands import extract, download_file
|
||||
|
||||
deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies")
|
||||
deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
|
||||
first_run = True
|
||||
|
||||
if self.force:
|
||||
if os.path.isdir(deps_dir):
|
||||
shutil.rmtree(deps_dir)
|
||||
|
||||
if not os.path.isdir(deps_dir):
|
||||
os.makedirs(deps_dir)
|
||||
|
||||
# Read file with installed dependencies, if exist
|
||||
installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt")
|
||||
if os.path.exists(installed_deps_file):
|
||||
installed_deps = [l.strip() for l in open(installed_deps_file)]
|
||||
else:
|
||||
installed_deps = []
|
||||
|
||||
# list of dependencies that need to be updated
|
||||
update_deps = list(set(packages) - set(installed_deps))
|
||||
|
||||
for dep in packages:
|
||||
dep_name = dep.split("-")[0]
|
||||
|
||||
# Don't download CMake if already exists in PATH
|
||||
if dep_name == "cmake":
|
||||
if spawn.find_executable(dep_name):
|
||||
continue
|
||||
|
||||
dep_dir = os.path.join(deps_dir, dep_name)
|
||||
# if not installed or need to be updated
|
||||
if not os.path.exists(dep_dir) or dep in update_deps:
|
||||
if first_run:
|
||||
print "Installing missing MSVC dependencies..."
|
||||
first_run = False
|
||||
|
||||
dep_version_dir = os.path.join(deps_dir, dep)
|
||||
|
||||
if os.path.exists(dep_version_dir):
|
||||
shutil.rmtree(dep_version_dir)
|
||||
|
||||
dep_zip = dep_version_dir + ".zip"
|
||||
if not os.path.isfile(dep_zip):
|
||||
download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip)
|
||||
|
||||
print "Extracting %s..." % dep,
|
||||
extract(dep_zip, deps_dir)
|
||||
print "done"
|
||||
|
||||
# Delete directory if exist
|
||||
if os.path.exists(dep_dir):
|
||||
shutil.rmtree(dep_dir)
|
||||
os.rename(dep_version_dir, dep_dir)
|
||||
|
||||
# Write in installed-dependencies.txt file
|
||||
with open(installed_deps_file, 'w') as installed_file:
|
||||
for line in packages:
|
||||
installed_file.write(line + "\n")
|
||||
|
||||
def install_mobile_android_packages(self):
|
||||
sys.exit('We do not support building Android on Windows. Sorry!')
|
||||
|
|
@ -24,7 +24,8 @@ from mach.decorators import (
|
|||
Command,
|
||||
)
|
||||
|
||||
from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, host_triple, find_dep_path_newest
|
||||
from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, find_dep_path_newest
|
||||
from servo.util import host_triple
|
||||
|
||||
|
||||
def format_duration(seconds):
|
||||
|
|
|
|||
|
|
@ -18,11 +18,12 @@ import subprocess
|
|||
from subprocess import PIPE
|
||||
import sys
|
||||
import tarfile
|
||||
import platform
|
||||
|
||||
import toml
|
||||
|
||||
from mach.registrar import Registrar
|
||||
import toml
|
||||
|
||||
from servo.packages import WINDOWS_MSVC as msvc_deps
|
||||
from servo.util import host_triple
|
||||
|
||||
BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
|
||||
|
||||
|
|
@ -107,51 +108,6 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
|
|||
os.rename(temp_file, dest_archive)
|
||||
|
||||
|
||||
def host_triple():
|
||||
os_type = platform.system().lower()
|
||||
if os_type == "linux":
|
||||
os_type = "unknown-linux-gnu"
|
||||
elif os_type == "darwin":
|
||||
os_type = "apple-darwin"
|
||||
elif os_type == "android":
|
||||
os_type = "linux-androideabi"
|
||||
elif os_type == "windows":
|
||||
# If we are in a Visual Studio environment, use msvc
|
||||
if os.getenv("PLATFORM") is not None:
|
||||
os_type = "pc-windows-msvc"
|
||||
elif os.getenv("MSYSTEM") is not None:
|
||||
os_type = "pc-windows-gnu"
|
||||
else:
|
||||
os_type = "unknown"
|
||||
elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
|
||||
os_type = "pc-windows-gnu"
|
||||
elif os_type == "freebsd":
|
||||
os_type = "unknown-freebsd"
|
||||
else:
|
||||
os_type = "unknown"
|
||||
|
||||
cpu_type = platform.machine().lower()
|
||||
if os_type.endswith("-msvc"):
|
||||
# vcvars*.bat should set it properly
|
||||
platform_env = os.environ.get("PLATFORM")
|
||||
if platform_env == "X86":
|
||||
cpu_type = "i686"
|
||||
elif platform_env == "X64":
|
||||
cpu_type = "x86_64"
|
||||
else:
|
||||
cpu_type = "unknown"
|
||||
elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
|
||||
cpu_type = "i686"
|
||||
elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
|
||||
cpu_type = "x86_64"
|
||||
elif cpu_type == "arm":
|
||||
cpu_type = "arm"
|
||||
else:
|
||||
cpu_type = "unknown"
|
||||
|
||||
return "%s-%s" % (cpu_type, os_type)
|
||||
|
||||
|
||||
def normalize_env(env):
|
||||
# There is a bug in subprocess where it doesn't like unicode types in
|
||||
# environment variables. Here, ensure all unicode are converted to
|
||||
|
|
@ -428,14 +384,18 @@ class CommandBase(object):
|
|||
if "msvc" in (target or host_triple()):
|
||||
msvc_x64 = "64" if "x86_64" in (target or host_triple()) else ""
|
||||
msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies")
|
||||
extra_path += [path.join(msvc_deps_dir, "cmake", "bin")]
|
||||
extra_path += [path.join(msvc_deps_dir, "ninja", "bin")]
|
||||
|
||||
def package_dir(package):
|
||||
return path.join(msvc_deps_dir, package, msvc_deps[package])
|
||||
|
||||
extra_path += [path.join(package_dir("cmake"), "bin")]
|
||||
extra_path += [path.join(package_dir("ninja"), "bin")]
|
||||
# Link openssl
|
||||
env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include")
|
||||
env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64)
|
||||
env["OPENSSL_INCLUDE_DIR"] = path.join(package_dir("openssl"), "include")
|
||||
env["OPENSSL_LIB_DIR"] = path.join(package_dir("openssl"), "lib" + msvc_x64)
|
||||
env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD"
|
||||
# Link moztools
|
||||
env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin")
|
||||
env["MOZTOOLS_PATH"] = path.join(package_dir("moztools"), "bin")
|
||||
|
||||
if is_windows():
|
||||
if not os.environ.get("NATIVE_WIN32_PYTHON"):
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# Listed all packages for different platforms in one file
|
||||
|
||||
WINDOWS_GNU = [
|
||||
WINDOWS_GNU = set([
|
||||
"diffutils",
|
||||
"make",
|
||||
"mingw-w64-x86_64-toolchain",
|
||||
"mingw-w64-x86_64-freetype",
|
||||
"mingw-w64-x86_64-icu",
|
||||
|
|
@ -12,17 +12,15 @@ WINDOWS_GNU = [
|
|||
"mingw-w64-x86_64-ca-certificates",
|
||||
"mingw-w64-x86_64-expat",
|
||||
"mingw-w64-x86_64-cmake",
|
||||
"tar",
|
||||
"diffutils",
|
||||
"patch",
|
||||
"patchutils",
|
||||
"make",
|
||||
"python2-setuptools",
|
||||
]
|
||||
"tar",
|
||||
])
|
||||
|
||||
WINDOWS_MSVC = [
|
||||
"cmake-3.6.1",
|
||||
"ninja-1.7.1",
|
||||
"openssl-1.0.1t-vs2015",
|
||||
"moztools-0.0.1-5",
|
||||
]
|
||||
WINDOWS_MSVC = {
|
||||
"cmake": "3.6.1",
|
||||
"moztools": "0.0.1-5",
|
||||
"ninja": "1.7.1",
|
||||
"openssl": "1.0.1t-vs2015",
|
||||
}
|
||||
|
|
@ -30,8 +30,9 @@ from mach.decorators import (
|
|||
|
||||
from servo.command_base import (
|
||||
BuildNotFound, CommandBase,
|
||||
call, cd, check_call, host_triple, set_osmesa_env,
|
||||
call, cd, check_call, set_osmesa_env,
|
||||
)
|
||||
from servo.util import host_triple
|
||||
|
||||
from wptrunner import wptcommandline
|
||||
from update import updatecommandline
|
||||
|
|
|
|||
151
python/servo/util.py
Normal file
151
python/servo/util.py
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
|
||||
# file at the top-level directory of this distribution.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
# option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import os.path as path
|
||||
import platform
|
||||
import sys
|
||||
from socket import error as socket_error
|
||||
import StringIO
|
||||
import tarfile
|
||||
import zipfile
|
||||
import urllib2
|
||||
|
||||
|
||||
def host_triple():
|
||||
os_type = platform.system().lower()
|
||||
if os_type == "linux":
|
||||
os_type = "unknown-linux-gnu"
|
||||
elif os_type == "darwin":
|
||||
os_type = "apple-darwin"
|
||||
elif os_type == "android":
|
||||
os_type = "linux-androideabi"
|
||||
elif os_type == "windows":
|
||||
# If we are in a Visual Studio environment, use msvc
|
||||
if os.getenv("PLATFORM") is not None:
|
||||
os_type = "pc-windows-msvc"
|
||||
elif os.getenv("MSYSTEM") is not None:
|
||||
os_type = "pc-windows-gnu"
|
||||
else:
|
||||
os_type = "unknown"
|
||||
elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
|
||||
os_type = "pc-windows-gnu"
|
||||
elif os_type == "freebsd":
|
||||
os_type = "unknown-freebsd"
|
||||
else:
|
||||
os_type = "unknown"
|
||||
|
||||
cpu_type = platform.machine().lower()
|
||||
if os_type.endswith("-msvc"):
|
||||
# vcvars*.bat should set it properly
|
||||
platform_env = os.environ.get("PLATFORM")
|
||||
if platform_env == "X86":
|
||||
cpu_type = "i686"
|
||||
elif platform_env == "X64":
|
||||
cpu_type = "x86_64"
|
||||
else:
|
||||
cpu_type = "unknown"
|
||||
elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
|
||||
cpu_type = "i686"
|
||||
elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
|
||||
cpu_type = "x86_64"
|
||||
elif cpu_type == "arm":
|
||||
cpu_type = "arm"
|
||||
else:
|
||||
cpu_type = "unknown"
|
||||
|
||||
return "{}-{}".format(cpu_type, os_type)
|
||||
|
||||
|
||||
def download(desc, src, writer, start_byte=0):
|
||||
if start_byte:
|
||||
print("Resuming download of {}...".format(desc))
|
||||
else:
|
||||
print("Downloading {}...".format(desc))
|
||||
dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
|
||||
|
||||
try:
|
||||
req = urllib2.Request(src)
|
||||
if start_byte:
|
||||
req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
|
||||
resp = urllib2.urlopen(req)
|
||||
|
||||
fsize = None
|
||||
if resp.info().getheader('Content-Length'):
|
||||
fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte
|
||||
|
||||
recved = start_byte
|
||||
chunk_size = 8192
|
||||
|
||||
while True:
|
||||
chunk = resp.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
recved += len(chunk)
|
||||
if not dumb:
|
||||
if fsize is not None:
|
||||
pct = recved * 100.0 / fsize
|
||||
print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
|
||||
|
||||
sys.stdout.flush()
|
||||
writer.write(chunk)
|
||||
|
||||
if not dumb:
|
||||
print()
|
||||
except urllib2.HTTPError, e:
|
||||
print("Download failed ({}): {} - {}".format(e.code, e.reason, src))
|
||||
if e.code == 403:
|
||||
print("No Rust compiler binary available for this platform. "
|
||||
"Please see https://github.com/servo/servo/#prerequisites")
|
||||
sys.exit(1)
|
||||
except urllib2.URLError, e:
|
||||
print("Error downloading {}: {}. The failing URL was: {}".format(desc, e.reason, src))
|
||||
sys.exit(1)
|
||||
except socket_error, e:
|
||||
print("Looks like there's a connectivity issue, check your Internet connection. {}".format(e))
|
||||
sys.exit(1)
|
||||
except KeyboardInterrupt:
|
||||
writer.flush()
|
||||
raise
|
||||
|
||||
|
||||
def download_bytes(desc, src):
|
||||
content_writer = StringIO.StringIO()
|
||||
download(desc, src, content_writer)
|
||||
return content_writer.getvalue()
|
||||
|
||||
|
||||
def download_file(desc, src, dst):
|
||||
tmp_path = dst + ".part"
|
||||
try:
|
||||
start_byte = path.getsize(tmp_path)
|
||||
with open(tmp_path, 'ab') as fd:
|
||||
download(desc, src, fd, start_byte=start_byte)
|
||||
except os.error:
|
||||
with open(tmp_path, 'wb') as fd:
|
||||
download(desc, src, fd)
|
||||
os.rename(tmp_path, dst)
|
||||
|
||||
|
||||
def extract(src, dst, movedir=None):
|
||||
if src.endswith(".zip"):
|
||||
zipfile.ZipFile(src).extractall(dst)
|
||||
else:
|
||||
tarfile.open(src).extractall(dst)
|
||||
|
||||
if movedir:
|
||||
for f in os.listdir(movedir):
|
||||
frm = path.join(movedir, f)
|
||||
to = path.join(dst, f)
|
||||
os.rename(frm, to)
|
||||
os.rmdir(movedir)
|
||||
|
||||
os.remove(src)
|
||||
Loading…
Add table
Add a link
Reference in a new issue