Start organizing platform-specific Python code

This starts to split platform-specific Python code into its own module,
which should help to tidy up our mach commands and make things more
reusable.
This commit is contained in:
Martin Robinson 2023-05-18 15:09:30 +02:00
parent e09f85e17b
commit 5be14ecc3c
14 changed files with 462 additions and 405 deletions

View file

@ -7,9 +7,10 @@ from __future__ import print_function, unicode_literals
import os
import platform
import sys
import shutil
from distutils.spawn import find_executable
from subprocess import Popen
import shutil
from tempfile import TemporaryFile
SEARCH_PATHS = [
@ -228,10 +229,6 @@ def _is_windows():
return sys.platform == 'win32'
class DummyContext(object):
pass
def is_firefox_checkout(topdir):
parentdir = os.path.normpath(os.path.join(topdir, '..'))
is_firefox = os.path.isfile(os.path.join(parentdir,
@ -244,14 +241,24 @@ def bootstrap_command_only(topdir):
# because the module requires non-standard python packages
_activate_virtualenv(topdir, is_firefox_checkout(topdir))
from servo.bootstrap import bootstrap
# We cannot import these modules until the virtual environment
# is active because they depend on modules installed via the
# virtual environment.
# pylint: disable=import-outside-toplevel
import servo.platform
import servo.util
# We are not set up yet, so we always use the default cache directory
# for the initial bootstrap.
# TODO(mrobinson): Why not just run the bootstrap command in this case?
try:
servo.platform.get().bootstrap(
servo.util.get_default_cache_dir(topdir), '-f' in sys.argv)
except NotImplementedError as exception:
print(exception)
return 1
context = DummyContext()
context.topdir = topdir
force = False
if len(sys.argv) == 3 and sys.argv[2] == "-f":
force = True
bootstrap(context, force)
return 0
@ -260,8 +267,6 @@ def bootstrap(topdir):
topdir = os.path.abspath(topdir)
len(sys.argv) > 1 and sys.argv[1] == "bootstrap"
# We don't support paths with Unicode characters for now
# https://github.com/servo/servo/issues/10002
try:

View file

@ -1,314 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function
from distutils.spawn import find_executable
from distutils.version import LooseVersion
import os
import distro
import subprocess
import six
import urllib
from os import path
from subprocess import PIPE
from zipfile import BadZipfile
import servo.packages as packages
from servo.util import extract, download_file, host_triple
from servo.gstreamer import macos_gst_root
def check_macos_gstreamer_lib():
try:
env = os.environ.copy()
gst_root = macos_gst_root()
env["PATH"] = path.join(gst_root, "bin")
env["PKG_CONFIG_PATH"] = path.join(gst_root, "lib", "pkgconfig")
has_gst = subprocess.call(["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"],
stdout=PIPE, stderr=PIPE, env=env) == 0
gst_lib_dir = subprocess.check_output(["pkg-config", "--variable=libdir", "gstreamer-1.0"],
env=env)
return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8'))
except FileNotFoundError:
return False
def check_gstreamer_lib():
return subprocess.call(["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
stdout=PIPE, stderr=PIPE) == 0
def run_as_root(command, force=False):
if os.geteuid() != 0:
command.insert(0, 'sudo')
if force:
command.append('-y')
return subprocess.call(command)
def install_linux_deps(context, pkgs_ubuntu, pkgs_fedora, pkgs_void, force):
install = False
pkgs = []
if context.distro in ['Ubuntu', 'Debian GNU/Linux']:
command = ['apt-get', 'install']
pkgs = pkgs_ubuntu
if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0:
install = True
elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora', 'Fedora Linux']:
installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
pkgs = pkgs_fedora
for p in pkgs:
command = ['dnf', 'install']
if "|{}".format(p) not in installed_pkgs:
install = True
break
elif context.distro == 'void':
installed_pkgs = str(subprocess.check_output(['xbps-query', '-l']))
pkgs = pkgs_void
for p in pkgs:
command = ['xbps-install', '-A']
if "ii {}-".format(p) not in installed_pkgs:
install = force = True
break
if not install:
return False
print("Installing missing dependencies...")
if run_as_root(command + pkgs, force) != 0:
raise Exception("Installation of dependencies failed.")
return True
def gstreamer(context, force=False):
cur = os.curdir
gstdir = os.path.join(cur, "support", "linux", "gstreamer")
if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
return True
return False
def bootstrap_gstreamer(context, force=False):
if not gstreamer(context, force):
print("gstreamer is already set up")
return 0
def linux(context, force=False):
# Please keep these in sync with the packages in README.md
pkgs_apt = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev',
'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++',
'build-essential', 'cmake', 'libssl-dev',
'liblzma-dev', 'libxmu6', 'libxmu-dev',
"libxcb-render0-dev", "libxcb-shape0-dev", "libxcb-xfixes0-dev",
'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev',
'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev',
'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev',
'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13',
'libunwind-dev', 'llvm-dev']
pkgs_dnf = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel',
'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel',
'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf',
'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel',
'rpm-build', 'openssl-devel', 'cmake',
'libXcursor-devel', 'libXmu-devel',
'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache',
'clang', 'clang-libs', 'llvm', 'autoconf213', 'python3-devel',
'gstreamer1-devel', 'gstreamer1-plugins-base-devel',
'gstreamer1-plugins-bad-free-devel', 'libjpeg-turbo-devel',
'zlib', 'libjpeg']
pkgs_xbps = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config',
'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel',
'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake',
'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel',
'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel',
'clang', 'gstreamer1-devel', 'autoconf213',
'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']
installed_something = install_linux_deps(context, pkgs_apt, pkgs_dnf,
pkgs_xbps, force)
if not check_gstreamer_lib():
installed_something |= gstreamer(context, force)
if not installed_something:
print("Dependencies were already installed!")
return 0
def windows_msvc(context, force=False):
'''Bootstrapper for MSVC building on Windows.'''
deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
deps_url = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/"
def version(package):
return packages.WINDOWS_MSVC[package]
def package_dir(package):
return os.path.join(deps_dir, package, version(package))
def check_cmake(version):
cmake_path = find_executable("cmake")
if cmake_path:
cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE)
cmake_version_output = six.ensure_str(cmake.stdout.read()).splitlines()[0]
cmake_version = cmake_version_output.replace("cmake version ", "")
if LooseVersion(cmake_version) >= LooseVersion(version):
return True
return False
def prepare_file(zip_path, full_spec):
if not os.path.isfile(zip_path):
zip_url = "{}{}.zip".format(deps_url, urllib.parse.quote(full_spec))
download_file(full_spec, zip_url, zip_path)
print("Extracting {}...".format(full_spec), end='')
try:
extract(zip_path, deps_dir)
except BadZipfile:
print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
os.remove(zip_path)
prepare_file(zip_path, full_spec)
else:
print("done")
to_install = {}
for package in packages.WINDOWS_MSVC:
# Don't install CMake if it already exists in PATH
if package == "cmake" and check_cmake(version("cmake")):
continue
if not os.path.isdir(package_dir(package)):
to_install[package] = version(package)
if not to_install:
return 0
print("Installing missing MSVC dependencies...")
for package in to_install:
full_spec = '{}-{}'.format(package, version(package))
parent_dir = os.path.dirname(package_dir(package))
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
zip_path = package_dir(package) + ".zip"
prepare_file(zip_path, full_spec)
extracted_path = os.path.join(deps_dir, full_spec)
os.rename(extracted_path, package_dir(package))
return 0
LINUX_SPECIFIC_BOOTSTRAPPERS = {
"gstreamer": bootstrap_gstreamer,
}
def get_linux_distribution():
distrib = six.ensure_str(distro.name())
version = six.ensure_str(distro.version())
if distrib in ['LinuxMint', 'Linux Mint', 'KDE neon']:
if '.' in version:
major, _ = version.split('.', 1)
else:
major = version
if major == '22':
base_version = '22.04'
elif major == '21':
base_version = '21.04'
elif major == '20':
base_version = '20.04'
elif major == '19':
base_version = '18.04'
elif major == '18':
base_version = '16.04'
else:
raise Exception('unsupported version of %s: %s' % (distrib, version))
distrib, version = 'Ubuntu', base_version
elif distrib == 'Pop!_OS':
if '.' in version:
major, _ = version.split('.', 1)
else:
major = version
if major == '22':
base_version = '22.04'
elif major == '21':
base_version = '21.04'
elif major == '20':
base_version = '20.04'
elif major == '19':
base_version = '18.04'
elif major == '18':
base_version = '16.04'
else:
raise Exception('unsupported version of %s: %s' % (distrib, version))
distrib, version = 'Ubuntu', base_version
elif distrib.lower() == 'elementary':
if version == '5.0':
base_version = '18.04'
elif version[0:3] == '0.4':
base_version = '16.04'
else:
raise Exception('unsupported version of %s: %s' % (distrib, version))
distrib, version = 'Ubuntu', base_version
elif distrib.lower() == 'ubuntu':
if version > '22.04':
print('WARNING: unsupported version of %s: %s' % (distrib, version))
# Fixme: we should allow checked/supported versions only
elif distrib.lower() not in [
'centos',
'centos linux',
'debian gnu/linux',
'fedora',
'fedora linux',
'void',
'nixos',
'arch',
'arch linux',
]:
raise Exception('mach bootstrap does not support %s, please file a bug' % distrib)
return distrib, version
def bootstrap(context, force=False, specific=None):
'''Dispatches to the right bootstrapping function for the OS.'''
bootstrapper = None
if "windows-msvc" in host_triple():
bootstrapper = windows_msvc
elif "linux-gnu" in host_triple():
distrib, version = get_linux_distribution()
if distrib.lower() == 'nixos':
print('NixOS does not need bootstrap, it will automatically enter a nix-shell')
print('Just run ./mach build')
print('')
print('You will need to run a nix-shell if you are trying to run any of the built binaries')
print('To enter the nix-shell manually use:')
print(' $ nix-shell etc/shell.nix')
return
context.distro = distrib
context.distro_version = version
bootstrapper = LINUX_SPECIFIC_BOOTSTRAPPERS.get(specific, linux)
if bootstrapper is None:
print('Bootstrap support is not yet available for your OS.')
return 1
return bootstrapper(context, force=force)

View file

@ -27,7 +27,8 @@ from mach.decorators import (
Command,
)
import servo.bootstrap as bootstrap
import servo.platform
from servo.command_base import CommandBase, cd, check_call
from servo.util import delete, download_bytes, download_file, extract, check_hash
@ -41,10 +42,15 @@ class MachCommands(CommandBase):
action='store_true',
help='Boostrap without confirmation')
def bootstrap(self, force=False):
# This entry point isn't actually invoked, ./mach bootstrap is directly
# called by mach (see mach_bootstrap.bootstrap_command_only) so that
# Note: This entry point isn't actually invoked by ./mach bootstrap.
# ./mach bootstrap calls mach_bootstrap.bootstrap_command_only so that
# it can install dependencies without needing mach's dependencies
return bootstrap.bootstrap(self.context, force=force)
try:
servo.platform.get().bootstrap(self.context.sharedir, force)
except NotImplementedError as exception:
print(exception)
return 1
return 0
@Command('bootstrap-gstreamer',
description='Set up a local copy of the gstreamer libraries (linux only).',
@ -53,7 +59,12 @@ class MachCommands(CommandBase):
action='store_true',
help='Boostrap without confirmation')
def bootstrap_gstreamer(self, force=False):
return bootstrap.bootstrap(self.context, force=force, specific="gstreamer")
try:
servo.platform.get().bootstrap_gstreamer(self.context.sharedir, force)
except NotImplementedError as exception:
print(exception)
return 1
return 0
@Command('bootstrap-android',
description='Install the Android SDK and NDK.',

View file

@ -35,7 +35,7 @@ from mach.registrar import Registrar
from mach_bootstrap import _get_exec_path
from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root
from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins
from servo.util import host_triple
from servo.platform import host_triple
@CommandProvider

View file

@ -33,14 +33,12 @@ from os import path
from subprocess import PIPE
import toml
import servo.platform
from xml.etree.ElementTree import XML
from servo.util import download_file
from .bootstrap import check_gstreamer_lib, check_macos_gstreamer_lib
from servo.util import download_file, get_default_cache_dir
from mach.decorators import CommandArgument
from mach.registrar import Registrar
from servo.packages import WINDOWS_MSVC as msvc_deps
from servo.util import host_triple
from servo.gstreamer import macos_gst_root
BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
@ -287,9 +285,7 @@ class CommandBase(object):
# Handle missing/default items
self.config.setdefault("tools", {})
default_cache_dir = os.environ.get("SERVO_CACHE_DIR",
path.join(context.topdir, ".servo"))
self.config["tools"].setdefault("cache-dir", default_cache_dir)
self.config["tools"].setdefault("cache-dir", get_default_cache_dir(context.topdir))
resolverelative("tools", "cache-dir")
default_cargo_home = os.environ.get("CARGO_HOME",
@ -545,23 +541,23 @@ class CommandBase(object):
if "media-dummy" in features:
return False
# MacOS always needs the GStreamer environment variable, but should
# also check that the Servo-specific version is downloaded and available.
if is_macosx():
if check_macos_gstreamer_lib():
# We override homebrew gstreamer if installed and
# always use pkgconfig from official gstreamer framework
if servo.platform.get().is_gstreamer_installed():
return True
else:
raise Exception("Official GStreamer framework not found (we need at least 1.21)."
"Please see installation instructions in README.md")
try:
if check_gstreamer_lib():
if servo.platform.get().is_gstreamer_installed():
return False
except Exception:
# Some systems don't have pkg-config; we can't probe in this case
# and must hope for the best
return False
effective_target = target or host_triple()
effective_target = target or servo.platform.host_triple()
if "x86_64" not in effective_target or "android" in effective_target:
# We don't build gstreamer for non-x86_64 / android yet
return False
@ -581,7 +577,7 @@ install them, let us know by filing a bug!")
"""Some commands, like test-wpt, don't use a full build env,
but may still need dynamic search paths. This command sets that up"""
if not android and self.needs_gstreamer_env(None, os.environ):
gstpath = gstreamer_root(host_triple(), os.environ, self.get_top_dir())
gstpath = gstreamer_root(servo.platform.host_triple(), os.environ, self.get_top_dir())
if gstpath is None:
return
os.environ["LD_LIBRARY_PATH"] = path.join(gstpath, "lib")
@ -590,10 +586,11 @@ install them, let us know by filing a bug!")
os.environ["GST_PLUGIN_SCANNER"] = path.join(gstpath, "libexec", "gstreamer-1.0", "gst-plugin-scanner")
def msvc_package_dir(self, package):
return path.join(self.context.sharedir, "msvc-dependencies", package, msvc_deps[package])
return path.join(self.context.sharedir, "msvc-dependencies", package,
servo.platform.windows.DEPENDENCIES[package])
def vs_dirs(self):
assert 'windows' in host_triple()
assert 'windows' in servo.platform.host_triple()
vsinstalldir = os.environ.get('VSINSTALLDIR')
vs_version = os.environ.get('VisualStudioVersion')
if vsinstalldir and vs_version:
@ -622,7 +619,7 @@ install them, let us know by filing a bug!")
env['PATH'] = env['PATH'].encode('ascii', 'ignore')
extra_path = []
extra_lib = []
if "msvc" in (target or host_triple()):
if "msvc" in (target or servo.platform.host_triple()):
extra_path += [path.join(self.msvc_package_dir("cmake"), "bin")]
extra_path += [path.join(self.msvc_package_dir("llvm"), "bin")]
extra_path += [path.join(self.msvc_package_dir("ninja"), "bin")]
@ -636,7 +633,7 @@ install them, let us know by filing a bug!")
env["TARGET_CFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP"
env["TARGET_CXXFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP"
arch = (target or host_triple()).split('-')[0]
arch = (target or servo.platform.host_triple()).split('-')[0]
vcpkg_arch = {
"x86_64": "x64-windows",
"i686": "x86-windows",
@ -672,8 +669,8 @@ install them, let us know by filing a bug!")
# Always build harfbuzz from source
env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
if is_build and self.needs_gstreamer_env(target or host_triple(), env, uwp, features):
gst_root = gstreamer_root(target or host_triple(), env, self.get_top_dir())
if is_build and self.needs_gstreamer_env(target or servo.platform.host_triple(), env, uwp, features):
gst_root = gstreamer_root(target or servo.platform.host_triple(), env, self.get_top_dir())
bin_path = path.join(gst_root, "bin")
lib_path = path.join(gst_root, "lib")
pkg_config_path = path.join(lib_path, "pkgconfig")
@ -1018,7 +1015,7 @@ install them, let us know by filing a bug!")
if self.context.bootstrapped:
return
target_platform = target or host_triple()
target_platform = target or servo.platform.host_triple()
# Always check if all needed MSVC dependencies are installed
if "msvc" in target_platform:

View file

@ -1,14 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at https://mozilla.org/MPL/2.0/.
WINDOWS_MSVC = {
"cmake": "3.14.3",
"llvm": "15.0.5",
"moztools": "3.2",
"ninja": "1.7.1",
"nuget": "08-08-2019",
"openssl": "111.3.0+1.1.1c-vs2017-2019-09-18",
"gstreamer-uwp": "1.16.0.5",
"openxr-loader-uwp": "1.0",
}

View file

@ -0,0 +1,64 @@
# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import platform
from .base import Base
from .windows import Windows
def host_platform():
os_type = platform.system().lower()
if os_type == "linux":
os_type = "unknown-linux-gnu"
elif os_type == "darwin":
os_type = "apple-darwin"
elif os_type == "android":
os_type = "linux-androideabi"
elif os_type == "windows":
os_type = "pc-windows-msvc"
elif os_type == "freebsd":
os_type = "unknown-freebsd"
else:
os_type = "unknown"
return os_type
def host_triple():
os_type = host_platform()
cpu_type = platform.machine().lower()
if cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
cpu_type = "i686"
elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
cpu_type = "x86_64"
elif cpu_type == "arm":
cpu_type = "arm"
elif cpu_type == "aarch64":
cpu_type = "aarch64"
else:
cpu_type = "unknown"
return f"{cpu_type}-{os_type}"
def get():
# We import the concrete platforms in if-statements here, because
# each one might have platform-specific imports which might not
# resolve on all platforms.
# TODO(mrobinson): We should do this for Windows too, once we
# stop relying on platform-specific code outside of this module.
# pylint: disable=import-outside-toplevel
if "windows-msvc" in host_triple():
return Windows()
if "linux-gnu" in host_triple():
from .linux import Linux
return Linux()
if "apple-darwin" in host_triple():
from .macos import MacOS
return MacOS()
return Base()

View file

@ -0,0 +1,34 @@
# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import subprocess
class Base:
def _platform_bootstrap(self, _cache_dir: str, _force: bool) -> bool:
raise NotImplementedError("Bootstrap installation detection not yet available.")
def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
raise NotImplementedError("GStreamer bootstrap support is not yet available for your OS.")
def _platform_is_gstreamer_installed(self) -> bool:
return subprocess.call(
["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
def bootstrap(self, cache_dir: str, force: bool):
if not self._platform_bootstrap(cache_dir, force):
print("Dependencies were already installed!")
def bootstrap_gstreamer(self, cache_dir: str, force: bool):
if not self._platform_bootstrap_gstreamer(cache_dir, force):
print("Dependencies were already installed!")
def is_gstreamer_installed(self) -> bool:
return self._platform_is_gstreamer_installed()

View file

@ -0,0 +1,171 @@
# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
import subprocess
from typing import Tuple
import distro
import six
from .base import Base
# Please keep these in sync with the packages in README.md
APT_PKGS = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev',
'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++',
'build-essential', 'cmake', 'libssl-dev',
'liblzma-dev', 'libxmu6', 'libxmu-dev',
"libxcb-render0-dev", "libxcb-shape0-dev", "libxcb-xfixes0-dev",
'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev',
'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev',
'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev',
'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13',
'libunwind-dev', 'llvm-dev']
DNF_PKGS = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel',
'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel',
'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf',
'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel',
'rpm-build', 'openssl-devel', 'cmake',
'libXcursor-devel', 'libXmu-devel',
'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache',
'clang', 'clang-libs', 'llvm', 'autoconf213', 'python3-devel',
'gstreamer1-devel', 'gstreamer1-plugins-base-devel',
'gstreamer1-plugins-bad-free-devel', 'libjpeg-turbo-devel',
'zlib', 'libjpeg']
XBPS_PKGS = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config',
'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel',
'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake',
'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel',
'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel',
'clang', 'gstreamer1-devel', 'autoconf213',
'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']
class Linux(Base):
def __init__(self):
(self.distro, self.version) = Linux.get_distro_and_version()
@staticmethod
def get_distro_and_version() -> Tuple[str, str]:
distrib = six.ensure_str(distro.name())
version = six.ensure_str(distro.version())
if distrib in ['LinuxMint', 'Linux Mint', 'KDE neon', 'Pop!_OS']:
if '.' in version:
major, _ = version.split('.', 1)
else:
major = version
distrib = 'Ubuntu'
if major == '22':
version = '22.04'
elif major == '21':
version = '21.04'
elif major == '20':
version = '20.04'
elif major == '19':
version = '18.04'
elif major == '18':
version = '16.04'
if distrib.lower() == 'elementary':
distrib = 'Ubuntu'
if version == '5.0':
version = '18.04'
elif version[0:3] == '0.4':
version = '16.04'
return (distrib, version)
def _platform_bootstrap(self, _cache_dir: str, force: bool) -> bool:
if self.distro.lower() == 'nixos':
print('NixOS does not need bootstrap, it will automatically enter a nix-shell')
print('Just run ./mach build')
print('')
print('You will need to run a nix-shell if you are trying '
'to run any of the built binaries')
print('To enter the nix-shell manually use:')
print(' $ nix-shell etc/shell.nix')
return False
if self.distro.lower() == 'ubuntu' and self.version > '22.04':
print(f"WARNING: unsupported version of {self.distro}: {self.version}")
# FIXME: Better version checking for these distributions.
if self.distro.lower() not in [
'arch linux',
'arch',
'centos linux',
'centos',
'debian gnu/linux',
'fedora linux',
'fedora',
'nixos',
'ubuntu',
'void',
]:
raise NotImplementedError("mach bootstrap does not support "
f"{self.distro}, please file a bug")
installed_something = self.install_non_gstreamer_dependencies(force)
installed_something |= self._platform_bootstrap_gstreamer(_cache_dir, force)
return installed_something
def install_non_gstreamer_dependencies(self, force: bool) -> bool:
install = False
pkgs = []
if self.distro in ['Ubuntu', 'Debian GNU/Linux']:
command = ['apt-get', 'install']
pkgs = APT_PKGS
if subprocess.call(['dpkg', '-s'] + pkgs,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) != 0:
install = True
elif self.distro in ['CentOS', 'CentOS Linux', 'Fedora', 'Fedora Linux']:
installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
pkgs = DNF_PKGS
for pkg in pkgs:
command = ['dnf', 'install']
if "|{}".format(pkg) not in installed_pkgs:
install = True
break
elif self.distro == 'void':
installed_pkgs = str(subprocess.check_output(['xbps-query', '-l']))
pkgs = XBPS_PKGS
for pkg in pkgs:
command = ['xbps-install', '-A']
if "ii {}-".format(pkg) not in installed_pkgs:
install = force = True
break
if not install:
return False
def run_as_root(command, force=False):
if os.geteuid() != 0:
command.insert(0, 'sudo')
if force:
command.append('-y')
return subprocess.call(command)
print("Installing missing dependencies...")
if run_as_root(command + pkgs, force) != 0:
raise Exception("Installation of dependencies failed.")
return True
def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
if self.is_gstreamer_installed():
return False
gstdir = os.path.join(os.curdir, "support", "linux", "gstreamer")
if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
return True
return False

View file

@ -0,0 +1,36 @@
# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
import subprocess
from .base import Base
from ..gstreamer import macos_gst_root
class MacOS(Base):
def __init__(self):
pass
def _platform_is_gstreamer_installed(self) -> bool:
# We override homebrew gstreamer if installed and always use pkgconfig
# from official gstreamer framework.
try:
gst_root = macos_gst_root()
env = os.environ.copy()
env["PATH"] = os.path.join(gst_root, "bin")
env["PKG_CONFIG_PATH"] = os.path.join(gst_root, "lib", "pkgconfig")
has_gst = subprocess.call(
["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) == 0
gst_lib_dir = subprocess.check_output(
["pkg-config", "--variable=libdir", "gstreamer-1.0"], env=env)
return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8'))
except FileNotFoundError:
return False

View file

@ -0,0 +1,98 @@
# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
import shutil
import subprocess
import urllib
import zipfile
from distutils.version import LooseVersion
import six
from .base import Base
from ..util import extract, download_file
DEPS_URL = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/"
DEPENDENCIES = {
"cmake": "3.14.3",
"llvm": "15.0.5",
"moztools": "3.2",
"ninja": "1.7.1",
"nuget": "08-08-2019",
"openssl": "111.3.0+1.1.1c-vs2017-2019-09-18",
"gstreamer-uwp": "1.16.0.5",
"openxr-loader-uwp": "1.0",
}
class Windows(Base):
def __init__(self):
pass
@staticmethod
def cmake_already_installed(required_version: str) -> bool:
cmake_path = shutil.which("cmake")
if not cmake_path:
return False
output = subprocess.check_output([cmake_path, "--version"])
cmake_version_output = six.ensure_str(output).splitlines()[0]
installed_version = cmake_version_output.replace("cmake version ", "")
return LooseVersion(installed_version) >= LooseVersion(required_version)
@classmethod
def prepare_file(cls, deps_dir: str, zip_path: str, full_spec: str):
if not os.path.isfile(zip_path):
zip_url = "{}{}.zip".format(DEPS_URL, urllib.parse.quote(full_spec))
download_file(full_spec, zip_url, zip_path)
print("Extracting {}...".format(full_spec), end='')
try:
extract(zip_path, deps_dir)
except zipfile.BadZipfile:
print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
os.remove(zip_path)
cls.prepare_file(deps_dir, zip_path, full_spec)
else:
print("done")
def _platform_bootstrap(self, cache_dir: str, _force: bool = False) -> bool:
deps_dir = os.path.join(cache_dir, "msvc-dependencies")
def get_package_dir(package, version) -> str:
return os.path.join(deps_dir, package, version)
to_install = {}
for (package, version) in DEPENDENCIES.items():
# Don't install CMake if it already exists in PATH
if package == "cmake" and self.cmake_already_installed(version):
continue
if not os.path.isdir(get_package_dir(package, version)):
to_install[package] = version
if not to_install:
return False
print("Installing missing MSVC dependencies...")
for (package, version) in to_install.items():
full_spec = '{}-{}'.format(package, version)
package_dir = get_package_dir(package, version)
parent_dir = os.path.dirname(package_dir)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
self.prepare_file(deps_dir, package_dir + ".zip", full_spec)
extracted_path = os.path.join(deps_dir, full_spec)
os.rename(extracted_path, package_dir)
return True

View file

@ -40,7 +40,7 @@ from servo.command_base import (
call, check_call, check_output,
)
from servo_tidy_tests import test_tidy
from servo.util import host_triple
from servo.platform import host_triple
SCRIPT_PATH = os.path.split(__file__)[0]
PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))

View file

@ -12,7 +12,6 @@ from __future__ import absolute_import, print_function, unicode_literals
import hashlib
import os
import os.path
import platform
import shutil
import stat
import sys
@ -59,40 +58,6 @@ def delete(path):
os.remove(path)
def host_platform():
os_type = platform.system().lower()
if os_type == "linux":
os_type = "unknown-linux-gnu"
elif os_type == "darwin":
os_type = "apple-darwin"
elif os_type == "android":
os_type = "linux-androideabi"
elif os_type == "windows":
os_type = "pc-windows-msvc"
elif os_type == "freebsd":
os_type = "unknown-freebsd"
else:
os_type = "unknown"
return os_type
def host_triple():
os_type = host_platform()
cpu_type = platform.machine().lower()
if cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
cpu_type = "i686"
elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
cpu_type = "x86_64"
elif cpu_type == "arm":
cpu_type = "arm"
elif cpu_type == "aarch64":
cpu_type = "aarch64"
else:
cpu_type = "unknown"
return "{}-{}".format(cpu_type, os_type)
def download(desc, src, writer, start_byte=0):
if start_byte:
print("Resuming download of {} ...".format(src))
@ -229,3 +194,7 @@ def check_hash(filename, expected, algorithm):
if hasher.hexdigest() != expected:
print("Incorrect {} hash for {}".format(algorithm, filename))
sys.exit(1)
def get_default_cache_dir(topdir):
return os.environ.get("SERVO_CACHE_DIR", os.path.join(topdir, ".servo"))

View file

@ -77,7 +77,7 @@ import os
import os.path as path
import re
import uuid
from servo.command_base import host_triple
from servo.platform import host_triple
def make_id(s):
s = s.replace("-", "_").replace("/", "_").replace("\\", "_")