mach: adopt uv and avoid system python (#34632)

This allows us to use `uv` for:
1. Installing a pinned Python version
2. Installing the dependency packages using `uv`'s pip compatible interface.
4. Bootstrapping `mach` without a Python installion on the host, using `uv
   run`

This change also introduces a new 'composite' GitHub action to setup
python in the different CI workflows. There is no support for externally
managed python installations and virtual environments. These could be
added in the future.

Fixes #34095, #34547

Signed-off-by: Mukilan Thiyagarajan <mukilan@igalia.com>
This commit is contained in:
Mukilan Thiyagarajan 2024-12-16 14:50:37 +05:30 committed by GitHub
parent f757fa46ac
commit 88a35b3cc9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 131 additions and 153 deletions

20
.github/actions/setup-python/action.yml vendored Normal file
View file

@ -0,0 +1,20 @@
name: Setup Python and uv
inputs:
skip-python-setup:
required: false
description: "Whether to skip installing python using Github's `setup-python` action"
default: false
runs:
using: "composite"
steps:
# Use the setup-python action to take advantage of the cache. uv will
# symlink to this version.
- name: Setup system python
if: ${{ inputs.skip-python-setup != 'true' }}
uses: actions/setup-python@v5
with:
python-version-file: '.python-version'
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.5.6"

View file

@ -61,8 +61,8 @@ jobs:
uses: mozilla-actions/sccache-action@v0.0.7
- name: Install crown
run: cargo install --path support/crown
- name: Bootstrap Python
run: python3 -m pip install --upgrade pip virtualenv
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap dependencies
run: sudo apt update && python3 ./mach bootstrap --skip-lints
- name: Set up JDK 17

View file

@ -13,9 +13,10 @@ jobs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap
run: |
python3 -m pip install --upgrade pip
sudo apt update
python3 ./mach bootstrap --skip-lints
- name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle.

View file

@ -33,9 +33,8 @@ jobs:
uses: mozilla-actions/sccache-action@v0.0.6
- name: Set LIBCLANG_PATH env # needed for bindgen in mozangle
run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Install taplo
uses: baptiste0928/cargo-install@v3
with:
@ -46,8 +45,6 @@ jobs:
with:
crate: cargo-deny
locked: true
- name: Bootstrap Python
run: python3 -m pip install --upgrade pip
- name: Bootstrap dependencies
run: |
sudo apt update
@ -57,4 +54,4 @@ jobs:
run: |
python3 ./mach clippy --use-crown --locked -- -- --deny warnings
- name: Tidy
run: python3 ./mach test-tidy --no-progress --all
run: python3 ./mach test-tidy --no-progress --all

View file

@ -35,9 +35,10 @@ jobs:
path: release-binary-linux
- name: unPackage binary
run: tar -xzf release-binary-linux/target.tar.gz
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap dependencies
run: |
python3 -m pip install --upgrade pip
sudo apt update
sudo apt install -qy --no-install-recommends mesa-vulkan-drivers
python3 ./mach bootstrap --skip-lints

View file

@ -52,9 +52,10 @@ jobs:
path: ${{ inputs.profile }}-binary-linux
- name: unPackage binary
run: tar -xzf ${{ inputs.profile }}-binary-linux/target.tar.gz
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap dependencies
run: |
python3 -m pip install --upgrade pip
sudo apt update
sudo apt install -qy --no-install-recommends mesa-vulkan-drivers
python3 ./mach bootstrap --skip-lints

View file

@ -141,13 +141,10 @@ jobs:
- name: Set LIBCLANG_PATH env # needed for bindgen in mozangle
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly
run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
- uses: actions/setup-python@v5
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
- name: Setup Python
uses: ./.github/actions/setup-python
with:
python-version: '3.10'
- name: Bootstrap Python
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
run: python3 -m pip install --upgrade pip
skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
- name: Bootstrap dependencies
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
run: |

View file

@ -41,15 +41,11 @@ jobs:
- uses: actions/download-artifact@v4
with:
name: ${{ inputs.profile }}-binary-macos
# Python 3.13 breaks wptrunner, so pin the version until
# web-platform-tests/wpt#48585 is fixed.
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Prep test environment
run: |
gtar -xzf target.tar.gz
python3 -m pip install --upgrade pip
python3 ./mach bootstrap --skip-lints
- name: Smoketest
run: python3 ./mach smoketest --${{ inputs.profile }}

View file

@ -79,18 +79,14 @@ jobs:
if: github.event_name == 'pull_request_target'
with:
ref: ${{ github.event.pull_request.head.sha }}
# Python 3.13 breaks wptrunner, so pin the version until
# web-platform-tests/wpt#48585 is fixed.
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.6
- name: Install crown
run: cargo install --path support/crown
- name: Bootstrap
run: |
python3 -m pip install --upgrade pip
python3 ./mach bootstrap --skip-lints
brew install gnu-tar
- name: Build (${{ inputs.profile }})

View file

@ -20,15 +20,16 @@ jobs:
outputs:
configuration: ${{ steps.configuration.outputs.result }}
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: actions/checkout@v4
with:
fetch-depth: 1
sparse-checkout: |
python/servo/try_parser.py
.github/actions/setup-python
.python-version
sparse-checkout-cone-mode: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Get Configuration
id: configuration
run: |

View file

@ -55,8 +55,8 @@ jobs:
uses: mozilla-actions/sccache-action@v0.0.6
- name: Install crown
run: cargo install --path support/crown
- name: Bootstrap Python
run: python3 -m pip install --upgrade pip virtualenv
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap dependencies
run: sudo apt update && python3 ./mach bootstrap --skip-lints
- name: Setup OpenHarmony SDK

View file

@ -28,10 +28,16 @@ jobs:
# using the token specified here.
# See https://github.com/actions/checkout/issues/162.
token: ${{ secrets.WPT_SYNC_TOKEN }}
- name: Setup Python
uses: ./servo/.github/actions/setup-python
- name: Install requirements
run: pip install -r servo/python/requirements.txt
run: |
uv venv
uv pip install -r servo/python/requirements.txt
- name: Process pull request
run: servo/python/wpt/export.py
run: |
source .venv/bin/activate
servo/python/wpt/export.py
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }}

View file

@ -35,9 +35,10 @@ jobs:
- uses: actions/download-artifact@v4
with:
name: wpt-full-logs-linux-layout-2020
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Prep environment
run: |
python3 -m pip install --upgrade pip
sudo apt update
python3 ./mach bootstrap
- name: Add upstream remote

View file

@ -77,14 +77,15 @@ jobs:
}
return try_string;
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: actions/checkout@v4
with:
sparse-checkout: |
python/servo/try_parser.py
.github/actions/setup-python
.python-version
sparse-checkout-cone-mode: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Parse Labels
if: ${{ steps.try_string.outputs.result }}
id: configuration

View file

@ -32,15 +32,16 @@ jobs:
outputs:
configuration: ${{ steps.configuration.outputs.result }}
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: actions/checkout@v4
with:
fetch-depth: 1
sparse-checkout: |
python/servo/try_parser.py
.github/actions/setup-python
.python-version
sparse-checkout-cone-mode: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Get Full Configuration
id: full_config
run: |

View file

@ -114,13 +114,13 @@ jobs:
choco install wixtoolset
echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- uses: actions/setup-python@v5
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
- name: Setup Python
uses: ./.github/actions/setup-python
with:
python-version: "3.10"
skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
- name: Bootstrap
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
run: |
python -m pip install --upgrade pip
python mach fetch
python mach bootstrap-gstreamer
# For some reason WiX isn't currently on the GitHub runner path. This is a

1
.python-version Normal file
View file

@ -0,0 +1 @@
3.12

View file

@ -17,7 +17,8 @@ For more detailed build instructions, see the Servo book under [Setting up your
### macOS
- Download and install [`python`](https://www.python.org/downloads/macos/) (version 3.10 to 3.12), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/).
- Download and install [Xcode](https://developer.apple.com/xcode/) and [`brew`](https://brew.sh/).
- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh`
- Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- Restart your shell to make sure `cargo` is available
- Install the other dependencies: `./mach bootstrap`
@ -25,11 +26,12 @@ For more detailed build instructions, see the Servo book under [Setting up your
### Linux
- Install `curl` and `python` (version 3.10 to 3.12):
- Arch: `sudo pacman -S --needed curl python python-pip`
- Debian, Ubuntu: `sudo apt install curl python3-pip python3-venv python3-setuptools`
- Fedora: `sudo dnf install curl python3 python3-pip python3-devel`
- Gentoo: `sudo emerge net-misc/curl dev-python/pip`
- Install `curl`:
- Arch: `sudo pacman -S --needed curl`
- Debian, Ubuntu: `sudo apt install curl`
- Fedora: `sudo dnf install curl`
- Gentoo: `sudo emerge net-misc/curl`
- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh`
- Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- Restart your shell to make sure `cargo` is available
- Install the other dependencies: `./mach bootstrap`
@ -37,7 +39,7 @@ For more detailed build instructions, see the Servo book under [Setting up your
### Windows
- Download and install [`python`](https://www.python.org/downloads/windows/) (version 3.10 to 3.12), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/)
- Download [`uv`](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/)
- Be sure to select *Quick install via the Visual Studio Community installer*
- In the Visual Studio Installer, ensure the following components are installed:
- **Windows 10 SDK (10.0.19041.0)** (`Microsoft.VisualStudio.Component.Windows10SDK.19041`)

52
mach
View file

@ -1,20 +1,34 @@
#!/usr/bin/env python3
#!/bin/sh
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# The beginning of this script is both valid shell and valid python, such that
# the script starts with the shell and is reexecuted with `uv run`. This
# ensures that the Python provided by the virtual environment (in the .venv
# directory) is used. If the virtual environment does not exist, `uv run` will
# still use the correct version of Python given in `.python-version` and
# python/mach_bootstrap.py will provision a new environment that will be used
# for the subsequent runs.
''':' && {
MACH_DIR=$(dirname "$0");
run_in_nix_if_needed() {
if { [ -f /etc/NIXOS ] || [ -n "${MACH_USE_NIX}" ]; } && [ -z "${IN_NIX_SHELL}" ]; then
EXTRA_NIX_ARGS=${SERVO_ANDROID_BUILD:+'--arg buildAndroid true'}
echo "NOTE: Entering nix-shell ${MACH_DIR}/shell.nix"
exec nix-shell "${MACH_DIR}/shell.nix" $EXTRA_NIX_ARGS --run "$*"
else
exec "$@"
fi
}
run_in_nix_if_needed uv run python ${MACH_DIR}/mach "$@"
}
'''
import os
import sys
# Destructure because version_info > max_ver is true when running the same version.
ver = (sys.version_info[0], sys.version_info[1])
min_ver = (3, 10)
max_ver = (3, 12) # WPT does not support Python 3.13. See issue #34095.
if ver < min_ver or ver > max_ver:
print("mach does not support python {0}.{1}, please install 3.{2} <= python <= 3.{3}" \
.format(ver[0], ver[1], min_ver[1], max_ver[1]))
sys.exit(1)
def main(args):
topdir = os.path.abspath(os.path.dirname(sys.argv[0]))
sys.path.insert(0, os.path.join(topdir, "python"))
@ -27,20 +41,4 @@ def main(args):
if __name__ == '__main__':
sys.dont_write_bytecode = True
need_nix_shell = os.path.exists('/etc/NIXOS') or 'MACH_USE_NIX' in os.environ
if need_nix_shell and not 'IN_NIX_SHELL' in os.environ:
import subprocess
from shlex import quote
mach_dir = os.path.abspath(os.path.dirname(__file__))
build_android_args = ['--arg', 'buildAndroid', 'true'] if 'SERVO_ANDROID_BUILD' in os.environ else []
print(f'NOTE: Entering nix-shell {mach_dir}/shell.nix')
try:
# sys argv already contains the ./mach part, so we just need to pass it as-is
result = subprocess.run(['nix-shell', f'{mach_dir}/shell.nix'] + build_android_args + ['--run', ' '.join(map(quote, sys.argv))])
sys.exit(result.returncode)
except KeyboardInterrupt:
sys.exit(0)
else:
main(sys.argv)
main(sys.argv)

View file

@ -1,9 +1,4 @@
@echo off
set workdir=%~dp0
where /Q py.exe
IF %ERRORLEVEL% NEQ 0 (
python %workdir%mach %*
) ELSE (
py -3 %workdir%mach %*
)
set workdir=%~dp0
uv run python %workdir%mach %*

View file

@ -4,10 +4,9 @@
import hashlib
import os
import platform
import site
import subprocess
import sys
import runpy
SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
TOP_DIR = os.path.abspath(os.path.join(SCRIPT_PATH, ".."))
@ -81,35 +80,16 @@ CATEGORIES = {
}
# venv calls its scripts folder "bin" on non-Windows and "Scripts" on Windows.
def _get_virtualenv_script_dir():
if os.name == "nt" and os.sep != "/":
return "Scripts"
return "bin"
# venv names its lib folder something like "lib/python3.11/site-packages" on
# non-Windows and "Lib\site-packages" on Window.
def _get_virtualenv_lib_dir():
if os.name == "nt" and os.sep != "/":
return os.path.join("Lib", "site-packages")
return os.path.join(
"lib",
f"python{sys.version_info[0]}.{sys.version_info[1]}",
"site-packages"
)
def _process_exec(args):
def _process_exec(args, cwd):
try:
subprocess.check_output(args, stderr=subprocess.STDOUT)
subprocess.check_output(args, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as exception:
print(exception.output.decode(sys.stdout.encoding))
print(f"Process failed with return code: {exception.returncode}")
sys.exit(1)
def install_virtual_env_requirements(project_path: str, python: str, virtualenv_path: str):
def install_virtual_env_requirements(project_path: str, virtualenv_path: str):
requirements_paths = [
os.path.join(project_path, "python", "requirements.txt"),
os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",),
@ -131,45 +111,32 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_
requirements_hash = requirements_hasher.hexdigest()
if marker_hash != requirements_hash:
print(" * Upgrading pip...")
_process_exec([python, "-m", "pip", "install", "--upgrade", "pip"])
print(" * Installing Python requirements...")
_process_exec([python, "-m", "pip", "install", "-I",
"-r", requirements_paths[0],
"-r", requirements_paths[1],
"-r", requirements_paths[2]])
pip_install_command = ["uv", "pip", "install"]
for requirements in requirements_paths:
pip_install_command.extend(["-r", requirements])
_process_exec(pip_install_command, cwd=project_path)
with open(marker_path, "w") as marker_file:
marker_file.write(requirements_hash)
def _activate_virtualenv(topdir):
virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1]))
python = sys.executable
virtualenv_path = os.path.join(topdir, ".venv")
if os.environ.get("VIRTUAL_ENV") != virtualenv_path:
venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir())
if not os.path.exists(virtualenv_path):
print(" * Setting up virtual environment...")
_process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path])
_process_exec(["uv", "venv"], cwd=topdir)
# This general approach is taken from virtualenv's `activate_this.py`.
os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)])
os.environ["VIRTUAL_ENV"] = virtualenv_path
script_dir = "Scripts" if _is_windows() else "bin"
runpy.run_path(os.path.join(virtualenv_path, script_dir, 'activate_this.py'))
prev_length = len(sys.path)
lib_path = os.path.realpath(os.path.join(virtualenv_path, _get_virtualenv_lib_dir()))
site.addsitedir(lib_path)
sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
install_virtual_env_requirements(topdir, virtualenv_path)
sys.real_prefix = sys.prefix
sys.prefix = virtualenv_path
# Use the python in our venv for subprocesses, not the python we were originally run with.
# Otherwise pip may still try to write to the wrong site-packages directory.
python = os.path.join(venv_script_path, "python")
install_virtual_env_requirements(topdir, python, virtualenv_path)
# Turn off warnings about deprecated syntax in our indirect dependencies.
# TODO: Find a better approach for doing this.
import warnings
warnings.filterwarnings('ignore', category=SyntaxWarning, module=r'.*.venv')
def _ensure_case_insensitive_if_windows():
@ -221,13 +188,6 @@ def bootstrap(topdir):
print('Current path:', topdir)
sys.exit(1)
# Ensure we are running Python 3.10+. We put this check here so we generate a
# user-friendly error message rather than a cryptic stack trace on module import.
if sys.version_info < (3, 10):
print('Python3 (>=3.10) is required to run mach.')
print('You are running Python', platform.python_version())
sys.exit(1)
_activate_virtualenv(topdir)
def populate_context(context, key=None):

View file

@ -175,7 +175,7 @@ class MachCommands(CommandBase):
return status
@Command('clean',
description='Clean the target/ and python/_venv[version]/ directories',
description='Clean the target/ and Python virtual environment directories',
category='build')
@CommandArgument('--manifest-path',
default=None,
@ -188,8 +188,7 @@ class MachCommands(CommandBase):
def clean(self, manifest_path=None, params=[], verbose=False):
self.ensure_bootstrapped()
virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1])
virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname)
virtualenv_path = path.join(self.get_top_dir(), '.venv')
if path.exists(virtualenv_path):
print('Removing virtualenv directory: %s' % virtualenv_path)
shutil.rmtree(virtualenv_path)

View file

@ -80,8 +80,7 @@ class PostBuildCommands(CommandBase):
@CommandBase.allow_target_configuration
def run(self, servo_binary: str, params, debugger=False, debugger_cmd=None,
headless=False, software=False, emulator=False, usb=False):
self._run(servo_binary, params, debugger, debugger_cmd,
headless, software, emulator, usb)
return self._run(servo_binary, params, debugger, debugger_cmd, headless, software, emulator, usb)
def _run(self, servo_binary: str, params, debugger=False, debugger_cmd=None,
headless=False, software=False, emulator=False, usb=False):

View file

@ -18,7 +18,7 @@ import subprocess
import textwrap
import json
from python.servo.post_build_commands import PostBuildCommands
from servo.post_build_commands import PostBuildCommands
import wpt
import wpt.manifestupdate
import wpt.run

View file

@ -41,7 +41,6 @@ directories = [
"./tests/wpt/mozilla/tests/mozilla/referrer-policy",
"./tests/wpt/mozilla/tests/webgl",
"./python/tidy/tests",
"./python/_v*",
"./python/mach",
# Generated and upstream code combined with our own. Could use cleanup
"./target",

View file

@ -5,7 +5,7 @@
buildAndroid ? false
}:
with import (builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/d04953086551086b44b6f3c6b7eeb26294f207da.tar.gz";
url = "https://github.com/NixOS/nixpkgs/archive/5d67ea6b4b63378b9c13be21e2ec9d1afc921713.tar.gz";
}) {
overlays = [
(import (builtins.fetchTarball {
@ -84,7 +84,13 @@ stdenv.mkDerivation (androidEnvironment // {
# Build utilities
cmake dbus gcc git pkg-config which llvm perl yasm m4
(python3.withPackages (ps: with ps; [virtualenv pip dbus]))
# Ensure the Python version is same as the one in `.python-version` file so
# that `uv` will just symlink to the one in nix store. Otherwise `uv` will
# download a pre-built binary that won't work on nix.
# FIXME: dbus python module needs to be installed into the virtual environment.
python312
uv
# This pins gnumake to 4.3 since 4.4 breaks jobserver
# functionality in mozjs and causes builds to be extremely