Start organizing platform-specific Python code

This starts to split platform-specific Python code into its own module,
which should help to tidy up our mach commands and make things more
reusable.

python/mach_bootstrap.py

@@ -7,9 +7,10 @@ from __future__ import print_function, unicode_literals
 import os
 import platform
 import sys
+import shutil
+
 from distutils.spawn import find_executable
 from subprocess import Popen
-import shutil
 from tempfile import TemporaryFile
 
 SEARCH_PATHS = [
@@ -228,10 +229,6 @@ def _is_windows():
     return sys.platform == 'win32'
 
 
-class DummyContext(object):
-    pass
-
-
 def is_firefox_checkout(topdir):
     parentdir = os.path.normpath(os.path.join(topdir, '..'))
     is_firefox = os.path.isfile(os.path.join(parentdir,
@@ -244,14 +241,24 @@ def bootstrap_command_only(topdir):
     # because the module requires non-standard python packages
     _activate_virtualenv(topdir, is_firefox_checkout(topdir))
 
-    from servo.bootstrap import bootstrap
+    # We cannot import these modules until the virtual environment
+    # is active because they depend on modules installed via the
+    # virtual environment.
+    # pylint: disable=import-outside-toplevel
+    import servo.platform
+    import servo.util
+
+    # We are not set up yet, so we always use the default cache directory
+    # for the initial bootstrap.
+    # TODO(mrobinson): Why not just run the bootstrap command in this case?
+
+    try:
+        servo.platform.get().bootstrap(
+            servo.util.get_default_cache_dir(topdir), '-f' in sys.argv)
+    except NotImplementedError as exception:
+        print(exception)
+        return 1
 
-    context = DummyContext()
-    context.topdir = topdir
-    force = False
-    if len(sys.argv) == 3 and sys.argv[2] == "-f":
-        force = True
-    bootstrap(context, force)
     return 0
 
 
@@ -260,8 +267,6 @@ def bootstrap(topdir):
 
     topdir = os.path.abspath(topdir)
 
-    len(sys.argv) > 1 and sys.argv[1] == "bootstrap"
-
     # We don't support paths with Unicode characters for now
     # https://github.com/servo/servo/issues/10002
     try:

python/servo/bootstrap.py

@@ -1,314 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function
-
-from distutils.spawn import find_executable
-from distutils.version import LooseVersion
-import os
-import distro
-import subprocess
-import six
-import urllib
-
-from os import path
-from subprocess import PIPE
-from zipfile import BadZipfile
-
-import servo.packages as packages
-from servo.util import extract, download_file, host_triple
-from servo.gstreamer import macos_gst_root
-
-
-def check_macos_gstreamer_lib():
-    try:
-        env = os.environ.copy()
-        gst_root = macos_gst_root()
-        env["PATH"] = path.join(gst_root, "bin")
-        env["PKG_CONFIG_PATH"] = path.join(gst_root, "lib", "pkgconfig")
-        has_gst = subprocess.call(["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"],
-                                  stdout=PIPE, stderr=PIPE, env=env) == 0
-        gst_lib_dir = subprocess.check_output(["pkg-config", "--variable=libdir", "gstreamer-1.0"],
-                                              env=env)
-        return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8'))
-    except FileNotFoundError:
-        return False
-
-
-def check_gstreamer_lib():
-    return subprocess.call(["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
-                           stdout=PIPE, stderr=PIPE) == 0
-
-
-def run_as_root(command, force=False):
-    if os.geteuid() != 0:
-        command.insert(0, 'sudo')
-    if force:
-        command.append('-y')
-    return subprocess.call(command)
-
-
-def install_linux_deps(context, pkgs_ubuntu, pkgs_fedora, pkgs_void, force):
-    install = False
-    pkgs = []
-    if context.distro in ['Ubuntu', 'Debian GNU/Linux']:
-        command = ['apt-get', 'install']
-        pkgs = pkgs_ubuntu
-        if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0:
-            install = True
-    elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora', 'Fedora Linux']:
-        installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
-        pkgs = pkgs_fedora
-        for p in pkgs:
-            command = ['dnf', 'install']
-            if "|{}".format(p) not in installed_pkgs:
-                install = True
-                break
-    elif context.distro == 'void':
-        installed_pkgs = str(subprocess.check_output(['xbps-query', '-l']))
-        pkgs = pkgs_void
-        for p in pkgs:
-            command = ['xbps-install', '-A']
-            if "ii {}-".format(p) not in installed_pkgs:
-                install = force = True
-                break
-
-    if not install:
-        return False
-
-    print("Installing missing dependencies...")
-    if run_as_root(command + pkgs, force) != 0:
-        raise Exception("Installation of dependencies failed.")
-    return True
-
-
-def gstreamer(context, force=False):
-    cur = os.curdir
-    gstdir = os.path.join(cur, "support", "linux", "gstreamer")
-    if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
-        subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
-        return True
-    return False
-
-
-def bootstrap_gstreamer(context, force=False):
-    if not gstreamer(context, force):
-        print("gstreamer is already set up")
-    return 0
-
-
-def linux(context, force=False):
-    # Please keep these in sync with the packages in README.md
-    pkgs_apt = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev',
-                'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++',
-                'build-essential', 'cmake', 'libssl-dev',
-                'liblzma-dev', 'libxmu6', 'libxmu-dev',
-                "libxcb-render0-dev", "libxcb-shape0-dev", "libxcb-xfixes0-dev",
-                'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev',
-                'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev',
-                'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev',
-                'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13',
-                'libunwind-dev', 'llvm-dev']
-    pkgs_dnf = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel',
-                'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel',
-                'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf',
-                'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel',
-                'rpm-build', 'openssl-devel', 'cmake',
-                'libXcursor-devel', 'libXmu-devel',
-                'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache',
-                'clang', 'clang-libs', 'llvm', 'autoconf213', 'python3-devel',
-                'gstreamer1-devel', 'gstreamer1-plugins-base-devel',
-                'gstreamer1-plugins-bad-free-devel', 'libjpeg-turbo-devel',
-                'zlib', 'libjpeg']
-    pkgs_xbps = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
-                 'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config',
-                 'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel',
-                 'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake',
-                 'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel',
-                 'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel',
-                 'clang', 'gstreamer1-devel', 'autoconf213',
-                 'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']
-
-    installed_something = install_linux_deps(context, pkgs_apt, pkgs_dnf,
-                                             pkgs_xbps, force)
-
-    if not check_gstreamer_lib():
-        installed_something |= gstreamer(context, force)
-
-    if not installed_something:
-        print("Dependencies were already installed!")
-
-    return 0
-
-
-def windows_msvc(context, force=False):
-    '''Bootstrapper for MSVC building on Windows.'''
-
-    deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
-    deps_url = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/"
-
-    def version(package):
-        return packages.WINDOWS_MSVC[package]
-
-    def package_dir(package):
-        return os.path.join(deps_dir, package, version(package))
-
-    def check_cmake(version):
-        cmake_path = find_executable("cmake")
-        if cmake_path:
-            cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE)
-            cmake_version_output = six.ensure_str(cmake.stdout.read()).splitlines()[0]
-            cmake_version = cmake_version_output.replace("cmake version ", "")
-            if LooseVersion(cmake_version) >= LooseVersion(version):
-                return True
-        return False
-
-    def prepare_file(zip_path, full_spec):
-        if not os.path.isfile(zip_path):
-            zip_url = "{}{}.zip".format(deps_url, urllib.parse.quote(full_spec))
-            download_file(full_spec, zip_url, zip_path)
-
-        print("Extracting {}...".format(full_spec), end='')
-        try:
-            extract(zip_path, deps_dir)
-        except BadZipfile:
-            print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
-            os.remove(zip_path)
-            prepare_file(zip_path, full_spec)
-        else:
-            print("done")
-
-    to_install = {}
-    for package in packages.WINDOWS_MSVC:
-        # Don't install CMake if it already exists in PATH
-        if package == "cmake" and check_cmake(version("cmake")):
-            continue
-
-        if not os.path.isdir(package_dir(package)):
-            to_install[package] = version(package)
-
-    if not to_install:
-        return 0
-
-    print("Installing missing MSVC dependencies...")
-    for package in to_install:
-        full_spec = '{}-{}'.format(package, version(package))
-
-        parent_dir = os.path.dirname(package_dir(package))
-        if not os.path.isdir(parent_dir):
-            os.makedirs(parent_dir)
-
-        zip_path = package_dir(package) + ".zip"
-        prepare_file(zip_path, full_spec)
-
-        extracted_path = os.path.join(deps_dir, full_spec)
-        os.rename(extracted_path, package_dir(package))
-
-    return 0
-
-
-LINUX_SPECIFIC_BOOTSTRAPPERS = {
-    "gstreamer": bootstrap_gstreamer,
-}
-
-
-def get_linux_distribution():
-    distrib = six.ensure_str(distro.name())
-    version = six.ensure_str(distro.version())
-
-    if distrib in ['LinuxMint', 'Linux Mint', 'KDE neon']:
-        if '.' in version:
-            major, _ = version.split('.', 1)
-        else:
-            major = version
-
-        if major == '22':
-            base_version = '22.04'
-        elif major == '21':
-            base_version = '21.04'
-        elif major == '20':
-            base_version = '20.04'
-        elif major == '19':
-            base_version = '18.04'
-        elif major == '18':
-            base_version = '16.04'
-        else:
-            raise Exception('unsupported version of %s: %s' % (distrib, version))
-
-        distrib, version = 'Ubuntu', base_version
-    elif distrib == 'Pop!_OS':
-        if '.' in version:
-            major, _ = version.split('.', 1)
-        else:
-            major = version
-
-        if major == '22':
-            base_version = '22.04'
-        elif major == '21':
-            base_version = '21.04'
-        elif major == '20':
-            base_version = '20.04'
-        elif major == '19':
-            base_version = '18.04'
-        elif major == '18':
-            base_version = '16.04'
-        else:
-            raise Exception('unsupported version of %s: %s' % (distrib, version))
-
-        distrib, version = 'Ubuntu', base_version
-    elif distrib.lower() == 'elementary':
-        if version == '5.0':
-            base_version = '18.04'
-        elif version[0:3] == '0.4':
-            base_version = '16.04'
-        else:
-            raise Exception('unsupported version of %s: %s' % (distrib, version))
-        distrib, version = 'Ubuntu', base_version
-    elif distrib.lower() == 'ubuntu':
-        if version > '22.04':
-            print('WARNING: unsupported version of %s: %s' % (distrib, version))
-    # Fixme: we should allow checked/supported versions only
-    elif distrib.lower() not in [
-        'centos',
-        'centos linux',
-        'debian gnu/linux',
-        'fedora',
-        'fedora linux',
-        'void',
-        'nixos',
-        'arch',
-        'arch linux',
-    ]:
-        raise Exception('mach bootstrap does not support %s, please file a bug' % distrib)
-
-    return distrib, version
-
-
-def bootstrap(context, force=False, specific=None):
-    '''Dispatches to the right bootstrapping function for the OS.'''
-
-    bootstrapper = None
-    if "windows-msvc" in host_triple():
-        bootstrapper = windows_msvc
-    elif "linux-gnu" in host_triple():
-        distrib, version = get_linux_distribution()
-
-        if distrib.lower() == 'nixos':
-            print('NixOS does not need bootstrap, it will automatically enter a nix-shell')
-            print('Just run ./mach build')
-            print('')
-            print('You will need to run a nix-shell if you are trying to run any of the built binaries')
-            print('To enter the nix-shell manually use:')
-            print('  $ nix-shell etc/shell.nix')
-            return
-
-        context.distro = distrib
-        context.distro_version = version
-        bootstrapper = LINUX_SPECIFIC_BOOTSTRAPPERS.get(specific, linux)
-
-    if bootstrapper is None:
-        print('Bootstrap support is not yet available for your OS.')
-        return 1
-
-    return bootstrapper(context, force=force)

python/servo/bootstrap_commands.py

@@ -27,7 +27,8 @@ from mach.decorators import (
     Command,
 )
 
-import servo.bootstrap as bootstrap
+import servo.platform
+
 from servo.command_base import CommandBase, cd, check_call
 from servo.util import delete, download_bytes, download_file, extract, check_hash
 
@@ -41,10 +42,15 @@ class MachCommands(CommandBase):
                      action='store_true',
                      help='Boostrap without confirmation')
     def bootstrap(self, force=False):
-        # This entry point isn't actually invoked, ./mach bootstrap is directly
+        # Note: This entry point isn't actually invoked by ./mach bootstrap.
-        # called by mach (see mach_bootstrap.bootstrap_command_only) so that
+        # ./mach bootstrap calls mach_bootstrap.bootstrap_command_only so that
         # it can install dependencies without needing mach's dependencies
-        return bootstrap.bootstrap(self.context, force=force)
+        try:
+            servo.platform.get().bootstrap(self.context.sharedir, force)
+        except NotImplementedError as exception:
+            print(exception)
+            return 1
+        return 0
 
     @Command('bootstrap-gstreamer',
              description='Set up a local copy of the gstreamer libraries (linux only).',
@@ -53,7 +59,12 @@ class MachCommands(CommandBase):
                      action='store_true',
                      help='Boostrap without confirmation')
     def bootstrap_gstreamer(self, force=False):
-        return bootstrap.bootstrap(self.context, force=force, specific="gstreamer")
+        try:
+            servo.platform.get().bootstrap_gstreamer(self.context.sharedir, force)
+        except NotImplementedError as exception:
+            print(exception)
+            return 1
+        return 0
 
     @Command('bootstrap-android',
              description='Install the Android SDK and NDK.',

python/servo/build_commands.py

@@ -35,7 +35,7 @@ from mach.registrar import Registrar
 from mach_bootstrap import _get_exec_path
 from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root
 from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins
-from servo.util import host_triple
+from servo.platform import host_triple
 
 
 @CommandProvider

python/servo/command_base.py

@@ -33,14 +33,12 @@ from os import path
 from subprocess import PIPE
 
 import toml
+import servo.platform
 
 from xml.etree.ElementTree import XML
-from servo.util import download_file
+from servo.util import download_file, get_default_cache_dir
-from .bootstrap import check_gstreamer_lib, check_macos_gstreamer_lib
 from mach.decorators import CommandArgument
 from mach.registrar import Registrar
-from servo.packages import WINDOWS_MSVC as msvc_deps
-from servo.util import host_triple
 from servo.gstreamer import macos_gst_root
 
 BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
@@ -287,9 +285,7 @@ class CommandBase(object):
 
         # Handle missing/default items
         self.config.setdefault("tools", {})
-        default_cache_dir = os.environ.get("SERVO_CACHE_DIR",
+        self.config["tools"].setdefault("cache-dir", get_default_cache_dir(context.topdir))
-                                           path.join(context.topdir, ".servo"))
-        self.config["tools"].setdefault("cache-dir", default_cache_dir)
         resolverelative("tools", "cache-dir")
 
         default_cargo_home = os.environ.get("CARGO_HOME",
@@ -545,23 +541,23 @@ class CommandBase(object):
         if "media-dummy" in features:
             return False
 
+        # MacOS always needs the GStreamer environment variable, but should
+        # also check that the Servo-specific version is downloaded and available.
         if is_macosx():
-            if check_macos_gstreamer_lib():
+            if servo.platform.get().is_gstreamer_installed():
-                # We override homebrew gstreamer if installed and
-                # always use pkgconfig from official gstreamer framework
                 return True
             else:
                 raise Exception("Official GStreamer framework not found (we need at least 1.21)."
                                 "Please see installation instructions in README.md")
 
         try:
-            if check_gstreamer_lib():
+            if servo.platform.get().is_gstreamer_installed():
                 return False
         except Exception:
             # Some systems don't have pkg-config; we can't probe in this case
             # and must hope for the best
             return False
-        effective_target = target or host_triple()
+        effective_target = target or servo.platform.host_triple()
         if "x86_64" not in effective_target or "android" in effective_target:
             # We don't build gstreamer for non-x86_64 / android yet
             return False
@@ -581,7 +577,7 @@ install them, let us know by filing a bug!")
         """Some commands, like test-wpt, don't use a full build env,
            but may still need dynamic search paths. This command sets that up"""
         if not android and self.needs_gstreamer_env(None, os.environ):
-            gstpath = gstreamer_root(host_triple(), os.environ, self.get_top_dir())
+            gstpath = gstreamer_root(servo.platform.host_triple(), os.environ, self.get_top_dir())
             if gstpath is None:
                 return
             os.environ["LD_LIBRARY_PATH"] = path.join(gstpath, "lib")
@@ -590,10 +586,11 @@ install them, let us know by filing a bug!")
             os.environ["GST_PLUGIN_SCANNER"] = path.join(gstpath, "libexec", "gstreamer-1.0", "gst-plugin-scanner")
 
     def msvc_package_dir(self, package):
-        return path.join(self.context.sharedir, "msvc-dependencies", package, msvc_deps[package])
+        return path.join(self.context.sharedir, "msvc-dependencies", package,
+                         servo.platform.windows.DEPENDENCIES[package])
 
     def vs_dirs(self):
-        assert 'windows' in host_triple()
+        assert 'windows' in servo.platform.host_triple()
         vsinstalldir = os.environ.get('VSINSTALLDIR')
         vs_version = os.environ.get('VisualStudioVersion')
         if vsinstalldir and vs_version:
@@ -622,7 +619,7 @@ install them, let us know by filing a bug!")
             env['PATH'] = env['PATH'].encode('ascii', 'ignore')
         extra_path = []
         extra_lib = []
-        if "msvc" in (target or host_triple()):
+        if "msvc" in (target or servo.platform.host_triple()):
             extra_path += [path.join(self.msvc_package_dir("cmake"), "bin")]
             extra_path += [path.join(self.msvc_package_dir("llvm"), "bin")]
             extra_path += [path.join(self.msvc_package_dir("ninja"), "bin")]
@@ -636,7 +633,7 @@ install them, let us know by filing a bug!")
                 env["TARGET_CFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP"
                 env["TARGET_CXXFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP"
 
-            arch = (target or host_triple()).split('-')[0]
+            arch = (target or servo.platform.host_triple()).split('-')[0]
             vcpkg_arch = {
                 "x86_64": "x64-windows",
                 "i686": "x86-windows",
@@ -672,8 +669,8 @@ install them, let us know by filing a bug!")
             # Always build harfbuzz from source
             env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
 
-        if is_build and self.needs_gstreamer_env(target or host_triple(), env, uwp, features):
+        if is_build and self.needs_gstreamer_env(target or servo.platform.host_triple(), env, uwp, features):
-            gst_root = gstreamer_root(target or host_triple(), env, self.get_top_dir())
+            gst_root = gstreamer_root(target or servo.platform.host_triple(), env, self.get_top_dir())
             bin_path = path.join(gst_root, "bin")
             lib_path = path.join(gst_root, "lib")
             pkg_config_path = path.join(lib_path, "pkgconfig")
@@ -1018,7 +1015,7 @@ install them, let us know by filing a bug!")
         if self.context.bootstrapped:
             return
 
-        target_platform = target or host_triple()
+        target_platform = target or servo.platform.host_triple()
 
         # Always check if all needed MSVC dependencies are installed
         if "msvc" in target_platform:

python/servo/packages.py

@@ -1,14 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at https://mozilla.org/MPL/2.0/.
-
-WINDOWS_MSVC = {
-    "cmake": "3.14.3",
-    "llvm": "15.0.5",
-    "moztools": "3.2",
-    "ninja": "1.7.1",
-    "nuget": "08-08-2019",
-    "openssl": "111.3.0+1.1.1c-vs2017-2019-09-18",
-    "gstreamer-uwp": "1.16.0.5",
-    "openxr-loader-uwp": "1.0",
-}

python/servo/platform/__init__.py

@@ -0,0 +1,64 @@
+# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import platform
+
+from .base import Base
+from .windows import Windows
+
+
+def host_platform():
+    os_type = platform.system().lower()
+    if os_type == "linux":
+        os_type = "unknown-linux-gnu"
+    elif os_type == "darwin":
+        os_type = "apple-darwin"
+    elif os_type == "android":
+        os_type = "linux-androideabi"
+    elif os_type == "windows":
+        os_type = "pc-windows-msvc"
+    elif os_type == "freebsd":
+        os_type = "unknown-freebsd"
+    else:
+        os_type = "unknown"
+    return os_type
+
+
+def host_triple():
+    os_type = host_platform()
+    cpu_type = platform.machine().lower()
+    if cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
+        cpu_type = "i686"
+    elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
+        cpu_type = "x86_64"
+    elif cpu_type == "arm":
+        cpu_type = "arm"
+    elif cpu_type == "aarch64":
+        cpu_type = "aarch64"
+    else:
+        cpu_type = "unknown"
+    return f"{cpu_type}-{os_type}"
+
+
+def get():
+    # We import the concrete platforms in if-statements here, because
+    # each one might have platform-specific imports which might not
+    # resolve on all platforms.
+    # TODO(mrobinson): We should do this for Windows too, once we
+    # stop relying on platform-specific code outside of this module.
+    # pylint: disable=import-outside-toplevel
+    if "windows-msvc" in host_triple():
+        return Windows()
+    if "linux-gnu" in host_triple():
+        from .linux import Linux
+        return Linux()
+    if "apple-darwin" in host_triple():
+        from .macos import MacOS
+        return MacOS()
+    return Base()

python/servo/platform/base.py

@@ -0,0 +1,34 @@
+# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import subprocess
+
+
+class Base:
+    def _platform_bootstrap(self, _cache_dir: str, _force: bool) -> bool:
+        raise NotImplementedError("Bootstrap installation detection not yet available.")
+
+    def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
+        raise NotImplementedError("GStreamer bootstrap support is not yet available for your OS.")
+
+    def _platform_is_gstreamer_installed(self) -> bool:
+        return subprocess.call(
+            ["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+
+    def bootstrap(self, cache_dir: str, force: bool):
+        if not self._platform_bootstrap(cache_dir, force):
+            print("Dependencies were already installed!")
+
+    def bootstrap_gstreamer(self, cache_dir: str, force: bool):
+        if not self._platform_bootstrap_gstreamer(cache_dir, force):
+            print("Dependencies were already installed!")
+
+    def is_gstreamer_installed(self) -> bool:
+        return self._platform_is_gstreamer_installed()

python/servo/platform/linux.py

@@ -0,0 +1,171 @@
+# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+import subprocess
+
+from typing import Tuple
+
+import distro
+import six
+
+from .base import Base
+
+# Please keep these in sync with the packages in README.md
+APT_PKGS = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev',
+            'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++',
+            'build-essential', 'cmake', 'libssl-dev',
+            'liblzma-dev', 'libxmu6', 'libxmu-dev',
+            "libxcb-render0-dev", "libxcb-shape0-dev", "libxcb-xfixes0-dev",
+            'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev',
+            'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev',
+            'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev',
+            'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13',
+            'libunwind-dev', 'llvm-dev']
+DNF_PKGS = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel',
+            'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel',
+            'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf',
+            'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel',
+            'rpm-build', 'openssl-devel', 'cmake',
+            'libXcursor-devel', 'libXmu-devel',
+            'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache',
+            'clang', 'clang-libs', 'llvm', 'autoconf213', 'python3-devel',
+            'gstreamer1-devel', 'gstreamer1-plugins-base-devel',
+            'gstreamer1-plugins-bad-free-devel', 'libjpeg-turbo-devel',
+            'zlib', 'libjpeg']
+XBPS_PKGS = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
+             'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config',
+             'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel',
+             'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake',
+             'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel',
+             'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel',
+             'clang', 'gstreamer1-devel', 'autoconf213',
+             'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']
+
+
+class Linux(Base):
+    def __init__(self):
+        (self.distro, self.version) = Linux.get_distro_and_version()
+
+    @staticmethod
+    def get_distro_and_version() -> Tuple[str, str]:
+        distrib = six.ensure_str(distro.name())
+        version = six.ensure_str(distro.version())
+
+        if distrib in ['LinuxMint', 'Linux Mint', 'KDE neon', 'Pop!_OS']:
+            if '.' in version:
+                major, _ = version.split('.', 1)
+            else:
+                major = version
+
+            distrib = 'Ubuntu'
+            if major == '22':
+                version = '22.04'
+            elif major == '21':
+                version = '21.04'
+            elif major == '20':
+                version = '20.04'
+            elif major == '19':
+                version = '18.04'
+            elif major == '18':
+                version = '16.04'
+
+        if distrib.lower() == 'elementary':
+            distrib = 'Ubuntu'
+            if version == '5.0':
+                version = '18.04'
+            elif version[0:3] == '0.4':
+                version = '16.04'
+
+        return (distrib, version)
+
+    def _platform_bootstrap(self, _cache_dir: str, force: bool) -> bool:
+        if self.distro.lower() == 'nixos':
+            print('NixOS does not need bootstrap, it will automatically enter a nix-shell')
+            print('Just run ./mach build')
+            print('')
+            print('You will need to run a nix-shell if you are trying '
+                  'to run any of the built binaries')
+            print('To enter the nix-shell manually use:')
+            print('  $ nix-shell etc/shell.nix')
+            return False
+
+        if self.distro.lower() == 'ubuntu' and self.version > '22.04':
+            print(f"WARNING: unsupported version of {self.distro}: {self.version}")
+
+        # FIXME: Better version checking for these distributions.
+        if self.distro.lower() not in [
+            'arch linux',
+            'arch',
+            'centos linux',
+            'centos',
+            'debian gnu/linux',
+            'fedora linux',
+            'fedora',
+            'nixos',
+            'ubuntu',
+            'void',
+        ]:
+            raise NotImplementedError("mach bootstrap does not support "
+                                      f"{self.distro}, please file a bug")
+
+        installed_something = self.install_non_gstreamer_dependencies(force)
+        installed_something |= self._platform_bootstrap_gstreamer(_cache_dir, force)
+        return installed_something
+
+    def install_non_gstreamer_dependencies(self, force: bool) -> bool:
+        install = False
+        pkgs = []
+        if self.distro in ['Ubuntu', 'Debian GNU/Linux']:
+            command = ['apt-get', 'install']
+            pkgs = APT_PKGS
+            if subprocess.call(['dpkg', '-s'] + pkgs,
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE) != 0:
+                install = True
+        elif self.distro in ['CentOS', 'CentOS Linux', 'Fedora', 'Fedora Linux']:
+            installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
+            pkgs = DNF_PKGS
+            for pkg in pkgs:
+                command = ['dnf', 'install']
+                if "|{}".format(pkg) not in installed_pkgs:
+                    install = True
+                    break
+        elif self.distro == 'void':
+            installed_pkgs = str(subprocess.check_output(['xbps-query', '-l']))
+            pkgs = XBPS_PKGS
+            for pkg in pkgs:
+                command = ['xbps-install', '-A']
+                if "ii {}-".format(pkg) not in installed_pkgs:
+                    install = force = True
+                    break
+
+        if not install:
+            return False
+
+        def run_as_root(command, force=False):
+            if os.geteuid() != 0:
+                command.insert(0, 'sudo')
+            if force:
+                command.append('-y')
+            return subprocess.call(command)
+
+        print("Installing missing dependencies...")
+        if run_as_root(command + pkgs, force) != 0:
+            raise Exception("Installation of dependencies failed.")
+        return True
+
+    def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
+        if self.is_gstreamer_installed():
+            return False
+
+        gstdir = os.path.join(os.curdir, "support", "linux", "gstreamer")
+        if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
+            subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
+            return True
+        return False

python/servo/platform/macos.py

@@ -0,0 +1,36 @@
+# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+import subprocess
+
+from .base import Base
+from ..gstreamer import macos_gst_root
+
+
+class MacOS(Base):
+    def __init__(self):
+        pass
+
+    def _platform_is_gstreamer_installed(self) -> bool:
+        # We override homebrew gstreamer if installed and always use pkgconfig
+        # from official gstreamer framework.
+        try:
+            gst_root = macos_gst_root()
+            env = os.environ.copy()
+            env["PATH"] = os.path.join(gst_root, "bin")
+            env["PKG_CONFIG_PATH"] = os.path.join(gst_root, "lib", "pkgconfig")
+            has_gst = subprocess.call(
+                ["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"],
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) == 0
+            gst_lib_dir = subprocess.check_output(
+                ["pkg-config", "--variable=libdir", "gstreamer-1.0"], env=env)
+            return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8'))
+        except FileNotFoundError:
+            return False

python/servo/platform/windows.py

@@ -0,0 +1,98 @@
+# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+import shutil
+import subprocess
+import urllib
+import zipfile
+
+from distutils.version import LooseVersion
+
+import six
+from .base import Base
+from ..util import extract, download_file
+
+DEPS_URL = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/"
+DEPENDENCIES = {
+    "cmake": "3.14.3",
+    "llvm": "15.0.5",
+    "moztools": "3.2",
+    "ninja": "1.7.1",
+    "nuget": "08-08-2019",
+    "openssl": "111.3.0+1.1.1c-vs2017-2019-09-18",
+    "gstreamer-uwp": "1.16.0.5",
+    "openxr-loader-uwp": "1.0",
+}
+
+
+class Windows(Base):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def cmake_already_installed(required_version: str) -> bool:
+        cmake_path = shutil.which("cmake")
+        if not cmake_path:
+            return False
+
+        output = subprocess.check_output([cmake_path, "--version"])
+        cmake_version_output = six.ensure_str(output).splitlines()[0]
+        installed_version = cmake_version_output.replace("cmake version ", "")
+        return LooseVersion(installed_version) >= LooseVersion(required_version)
+
+    @classmethod
+    def prepare_file(cls, deps_dir: str, zip_path: str, full_spec: str):
+        if not os.path.isfile(zip_path):
+            zip_url = "{}{}.zip".format(DEPS_URL, urllib.parse.quote(full_spec))
+            download_file(full_spec, zip_url, zip_path)
+
+        print("Extracting {}...".format(full_spec), end='')
+        try:
+            extract(zip_path, deps_dir)
+        except zipfile.BadZipfile:
+            print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
+            os.remove(zip_path)
+            cls.prepare_file(deps_dir, zip_path, full_spec)
+        else:
+            print("done")
+
+    def _platform_bootstrap(self, cache_dir: str, _force: bool = False) -> bool:
+        deps_dir = os.path.join(cache_dir, "msvc-dependencies")
+
+        def get_package_dir(package, version) -> str:
+            return os.path.join(deps_dir, package, version)
+
+        to_install = {}
+        for (package, version) in DEPENDENCIES.items():
+            # Don't install CMake if it already exists in PATH
+            if package == "cmake" and self.cmake_already_installed(version):
+                continue
+
+            if not os.path.isdir(get_package_dir(package, version)):
+                to_install[package] = version
+
+        if not to_install:
+            return False
+
+        print("Installing missing MSVC dependencies...")
+        for (package, version) in to_install.items():
+            full_spec = '{}-{}'.format(package, version)
+
+            package_dir = get_package_dir(package, version)
+            parent_dir = os.path.dirname(package_dir)
+            if not os.path.isdir(parent_dir):
+                os.makedirs(parent_dir)
+
+            self.prepare_file(deps_dir, package_dir + ".zip", full_spec)
+
+            extracted_path = os.path.join(deps_dir, full_spec)
+            os.rename(extracted_path, package_dir)
+
+        return True

python/servo/testing_commands.py

@@ -40,7 +40,7 @@ from servo.command_base import (
     call, check_call, check_output,
 )
 from servo_tidy_tests import test_tidy
-from servo.util import host_triple
+from servo.platform import host_triple
 
 SCRIPT_PATH = os.path.split(__file__)[0]
 PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))

python/servo/util.py

@@ -12,7 +12,6 @@ from __future__ import absolute_import, print_function, unicode_literals
 import hashlib
 import os
 import os.path
-import platform
 import shutil
 import stat
 import sys
@@ -59,40 +58,6 @@ def delete(path):
         os.remove(path)
 
 
-def host_platform():
-    os_type = platform.system().lower()
-    if os_type == "linux":
-        os_type = "unknown-linux-gnu"
-    elif os_type == "darwin":
-        os_type = "apple-darwin"
-    elif os_type == "android":
-        os_type = "linux-androideabi"
-    elif os_type == "windows":
-        os_type = "pc-windows-msvc"
-    elif os_type == "freebsd":
-        os_type = "unknown-freebsd"
-    else:
-        os_type = "unknown"
-    return os_type
-
-
-def host_triple():
-    os_type = host_platform()
-    cpu_type = platform.machine().lower()
-    if cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
-        cpu_type = "i686"
-    elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
-        cpu_type = "x86_64"
-    elif cpu_type == "arm":
-        cpu_type = "arm"
-    elif cpu_type == "aarch64":
-        cpu_type = "aarch64"
-    else:
-        cpu_type = "unknown"
-
-    return "{}-{}".format(cpu_type, os_type)
-
-
 def download(desc, src, writer, start_byte=0):
     if start_byte:
         print("Resuming download of {} ...".format(src))
@@ -229,3 +194,7 @@ def check_hash(filename, expected, algorithm):
     if hasher.hexdigest() != expected:
         print("Incorrect {} hash for {}".format(algorithm, filename))
         sys.exit(1)
+
+
+def get_default_cache_dir(topdir):
+    return os.environ.get("SERVO_CACHE_DIR", os.path.join(topdir, ".servo"))

support/windows/Servo.wxs.mako

@@ -77,7 +77,7 @@ import os
 import os.path as path
 import re
 import uuid
-from servo.command_base import host_triple
+from servo.platform import host_triple
 
 def make_id(s):
     s = s.replace("-", "_").replace("/", "_").replace("\\", "_")