mach: adopt uv and avoid system python (#34632)

This allows us to use `uv` for:
1. Installing a pinned Python version
2. Installing the dependency packages using `uv`'s pip compatible interface.
4. Bootstrapping `mach` without a Python installion on the host, using `uv
   run`

This change also introduces a new 'composite' GitHub action to setup
python in the different CI workflows. There is no support for externally
managed python installations and virtual environments. These could be
added in the future.

Fixes #34095, #34547

Signed-off-by: Mukilan Thiyagarajan <mukilan@igalia.com>
This commit is contained in:
Mukilan Thiyagarajan 2024-12-16 14:50:37 +05:30 committed by GitHub
parent f757fa46ac
commit 88a35b3cc9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 131 additions and 153 deletions

20
.github/actions/setup-python/action.yml vendored Normal file
View file

@ -0,0 +1,20 @@
name: Setup Python and uv
inputs:
skip-python-setup:
required: false
description: "Whether to skip installing python using Github's `setup-python` action"
default: false
runs:
using: "composite"
steps:
# Use the setup-python action to take advantage of the cache. uv will
# symlink to this version.
- name: Setup system python
if: ${{ inputs.skip-python-setup != 'true' }}
uses: actions/setup-python@v5
with:
python-version-file: '.python-version'
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.5.6"

View file

@ -61,8 +61,8 @@ jobs:
uses: mozilla-actions/sccache-action@v0.0.7 uses: mozilla-actions/sccache-action@v0.0.7
- name: Install crown - name: Install crown
run: cargo install --path support/crown run: cargo install --path support/crown
- name: Bootstrap Python - name: Setup Python
run: python3 -m pip install --upgrade pip virtualenv uses: ./.github/actions/setup-python
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: sudo apt update && python3 ./mach bootstrap --skip-lints run: sudo apt update && python3 ./mach bootstrap --skip-lints
- name: Set up JDK 17 - name: Set up JDK 17

View file

@ -13,9 +13,10 @@ jobs:
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap - name: Bootstrap
run: | run: |
python3 -m pip install --upgrade pip
sudo apt update sudo apt update
python3 ./mach bootstrap --skip-lints python3 ./mach bootstrap --skip-lints
- name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle. - name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle.

View file

@ -33,9 +33,8 @@ jobs:
uses: mozilla-actions/sccache-action@v0.0.6 uses: mozilla-actions/sccache-action@v0.0.6
- name: Set LIBCLANG_PATH env # needed for bindgen in mozangle - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle
run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
- uses: actions/setup-python@v5 - name: Setup Python
with: uses: ./.github/actions/setup-python
python-version: '3.10'
- name: Install taplo - name: Install taplo
uses: baptiste0928/cargo-install@v3 uses: baptiste0928/cargo-install@v3
with: with:
@ -46,8 +45,6 @@ jobs:
with: with:
crate: cargo-deny crate: cargo-deny
locked: true locked: true
- name: Bootstrap Python
run: python3 -m pip install --upgrade pip
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: | run: |
sudo apt update sudo apt update
@ -57,4 +54,4 @@ jobs:
run: | run: |
python3 ./mach clippy --use-crown --locked -- -- --deny warnings python3 ./mach clippy --use-crown --locked -- -- --deny warnings
- name: Tidy - name: Tidy
run: python3 ./mach test-tidy --no-progress --all run: python3 ./mach test-tidy --no-progress --all

View file

@ -35,9 +35,10 @@ jobs:
path: release-binary-linux path: release-binary-linux
- name: unPackage binary - name: unPackage binary
run: tar -xzf release-binary-linux/target.tar.gz run: tar -xzf release-binary-linux/target.tar.gz
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: | run: |
python3 -m pip install --upgrade pip
sudo apt update sudo apt update
sudo apt install -qy --no-install-recommends mesa-vulkan-drivers sudo apt install -qy --no-install-recommends mesa-vulkan-drivers
python3 ./mach bootstrap --skip-lints python3 ./mach bootstrap --skip-lints

View file

@ -52,9 +52,10 @@ jobs:
path: ${{ inputs.profile }}-binary-linux path: ${{ inputs.profile }}-binary-linux
- name: unPackage binary - name: unPackage binary
run: tar -xzf ${{ inputs.profile }}-binary-linux/target.tar.gz run: tar -xzf ${{ inputs.profile }}-binary-linux/target.tar.gz
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: | run: |
python3 -m pip install --upgrade pip
sudo apt update sudo apt update
sudo apt install -qy --no-install-recommends mesa-vulkan-drivers sudo apt install -qy --no-install-recommends mesa-vulkan-drivers
python3 ./mach bootstrap --skip-lints python3 ./mach bootstrap --skip-lints

View file

@ -141,13 +141,10 @@ jobs:
- name: Set LIBCLANG_PATH env # needed for bindgen in mozangle - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly
run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
- uses: actions/setup-python@v5 - name: Setup Python
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} uses: ./.github/actions/setup-python
with: with:
python-version: '3.10' skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
- name: Bootstrap Python
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
run: python3 -m pip install --upgrade pip
- name: Bootstrap dependencies - name: Bootstrap dependencies
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
run: | run: |

View file

@ -41,15 +41,11 @@ jobs:
- uses: actions/download-artifact@v4 - uses: actions/download-artifact@v4
with: with:
name: ${{ inputs.profile }}-binary-macos name: ${{ inputs.profile }}-binary-macos
# Python 3.13 breaks wptrunner, so pin the version until - name: Setup Python
# web-platform-tests/wpt#48585 is fixed. uses: ./.github/actions/setup-python
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Prep test environment - name: Prep test environment
run: | run: |
gtar -xzf target.tar.gz gtar -xzf target.tar.gz
python3 -m pip install --upgrade pip
python3 ./mach bootstrap --skip-lints python3 ./mach bootstrap --skip-lints
- name: Smoketest - name: Smoketest
run: python3 ./mach smoketest --${{ inputs.profile }} run: python3 ./mach smoketest --${{ inputs.profile }}

View file

@ -79,18 +79,14 @@ jobs:
if: github.event_name == 'pull_request_target' if: github.event_name == 'pull_request_target'
with: with:
ref: ${{ github.event.pull_request.head.sha }} ref: ${{ github.event.pull_request.head.sha }}
# Python 3.13 breaks wptrunner, so pin the version until - name: Setup Python
# web-platform-tests/wpt#48585 is fixed. uses: ./.github/actions/setup-python
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Run sccache-cache - name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.6 uses: mozilla-actions/sccache-action@v0.0.6
- name: Install crown - name: Install crown
run: cargo install --path support/crown run: cargo install --path support/crown
- name: Bootstrap - name: Bootstrap
run: | run: |
python3 -m pip install --upgrade pip
python3 ./mach bootstrap --skip-lints python3 ./mach bootstrap --skip-lints
brew install gnu-tar brew install gnu-tar
- name: Build (${{ inputs.profile }}) - name: Build (${{ inputs.profile }})

View file

@ -20,15 +20,16 @@ jobs:
outputs: outputs:
configuration: ${{ steps.configuration.outputs.result }} configuration: ${{ steps.configuration.outputs.result }}
steps: steps:
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
fetch-depth: 1 fetch-depth: 1
sparse-checkout: | sparse-checkout: |
python/servo/try_parser.py python/servo/try_parser.py
.github/actions/setup-python
.python-version
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Get Configuration - name: Get Configuration
id: configuration id: configuration
run: | run: |

View file

@ -55,8 +55,8 @@ jobs:
uses: mozilla-actions/sccache-action@v0.0.6 uses: mozilla-actions/sccache-action@v0.0.6
- name: Install crown - name: Install crown
run: cargo install --path support/crown run: cargo install --path support/crown
- name: Bootstrap Python - name: Setup Python
run: python3 -m pip install --upgrade pip virtualenv uses: ./.github/actions/setup-python
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: sudo apt update && python3 ./mach bootstrap --skip-lints run: sudo apt update && python3 ./mach bootstrap --skip-lints
- name: Setup OpenHarmony SDK - name: Setup OpenHarmony SDK

View file

@ -28,10 +28,16 @@ jobs:
# using the token specified here. # using the token specified here.
# See https://github.com/actions/checkout/issues/162. # See https://github.com/actions/checkout/issues/162.
token: ${{ secrets.WPT_SYNC_TOKEN }} token: ${{ secrets.WPT_SYNC_TOKEN }}
- name: Setup Python
uses: ./servo/.github/actions/setup-python
- name: Install requirements - name: Install requirements
run: pip install -r servo/python/requirements.txt run: |
uv venv
uv pip install -r servo/python/requirements.txt
- name: Process pull request - name: Process pull request
run: servo/python/wpt/export.py run: |
source .venv/bin/activate
servo/python/wpt/export.py
env: env:
GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_CONTEXT: ${{ toJson(github) }}
WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }} WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }}

View file

@ -35,9 +35,10 @@ jobs:
- uses: actions/download-artifact@v4 - uses: actions/download-artifact@v4
with: with:
name: wpt-full-logs-linux-layout-2020 name: wpt-full-logs-linux-layout-2020
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Prep environment - name: Prep environment
run: | run: |
python3 -m pip install --upgrade pip
sudo apt update sudo apt update
python3 ./mach bootstrap python3 ./mach bootstrap
- name: Add upstream remote - name: Add upstream remote

View file

@ -77,14 +77,15 @@ jobs:
} }
return try_string; return try_string;
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
sparse-checkout: | sparse-checkout: |
python/servo/try_parser.py python/servo/try_parser.py
.github/actions/setup-python
.python-version
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Parse Labels - name: Parse Labels
if: ${{ steps.try_string.outputs.result }} if: ${{ steps.try_string.outputs.result }}
id: configuration id: configuration

View file

@ -32,15 +32,16 @@ jobs:
outputs: outputs:
configuration: ${{ steps.configuration.outputs.result }} configuration: ${{ steps.configuration.outputs.result }}
steps: steps:
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
fetch-depth: 1 fetch-depth: 1
sparse-checkout: | sparse-checkout: |
python/servo/try_parser.py python/servo/try_parser.py
.github/actions/setup-python
.python-version
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Get Full Configuration - name: Get Full Configuration
id: full_config id: full_config
run: | run: |

View file

@ -114,13 +114,13 @@ jobs:
choco install wixtoolset choco install wixtoolset
echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- uses: actions/setup-python@v5 - uses: actions/setup-python@v5
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} - name: Setup Python
uses: ./.github/actions/setup-python
with: with:
python-version: "3.10" skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
- name: Bootstrap - name: Bootstrap
if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
run: | run: |
python -m pip install --upgrade pip
python mach fetch python mach fetch
python mach bootstrap-gstreamer python mach bootstrap-gstreamer
# For some reason WiX isn't currently on the GitHub runner path. This is a # For some reason WiX isn't currently on the GitHub runner path. This is a

1
.python-version Normal file
View file

@ -0,0 +1 @@
3.12

View file

@ -17,7 +17,8 @@ For more detailed build instructions, see the Servo book under [Setting up your
### macOS ### macOS
- Download and install [`python`](https://www.python.org/downloads/macos/) (version 3.10 to 3.12), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/). - Download and install [Xcode](https://developer.apple.com/xcode/) and [`brew`](https://brew.sh/).
- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh`
- Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- Restart your shell to make sure `cargo` is available - Restart your shell to make sure `cargo` is available
- Install the other dependencies: `./mach bootstrap` - Install the other dependencies: `./mach bootstrap`
@ -25,11 +26,12 @@ For more detailed build instructions, see the Servo book under [Setting up your
### Linux ### Linux
- Install `curl` and `python` (version 3.10 to 3.12): - Install `curl`:
- Arch: `sudo pacman -S --needed curl python python-pip` - Arch: `sudo pacman -S --needed curl`
- Debian, Ubuntu: `sudo apt install curl python3-pip python3-venv python3-setuptools` - Debian, Ubuntu: `sudo apt install curl`
- Fedora: `sudo dnf install curl python3 python3-pip python3-devel` - Fedora: `sudo dnf install curl`
- Gentoo: `sudo emerge net-misc/curl dev-python/pip` - Gentoo: `sudo emerge net-misc/curl`
- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh`
- Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- Restart your shell to make sure `cargo` is available - Restart your shell to make sure `cargo` is available
- Install the other dependencies: `./mach bootstrap` - Install the other dependencies: `./mach bootstrap`
@ -37,7 +39,7 @@ For more detailed build instructions, see the Servo book under [Setting up your
### Windows ### Windows
- Download and install [`python`](https://www.python.org/downloads/windows/) (version 3.10 to 3.12), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/) - Download [`uv`](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/)
- Be sure to select *Quick install via the Visual Studio Community installer* - Be sure to select *Quick install via the Visual Studio Community installer*
- In the Visual Studio Installer, ensure the following components are installed: - In the Visual Studio Installer, ensure the following components are installed:
- **Windows 10 SDK (10.0.19041.0)** (`Microsoft.VisualStudio.Component.Windows10SDK.19041`) - **Windows 10 SDK (10.0.19041.0)** (`Microsoft.VisualStudio.Component.Windows10SDK.19041`)

52
mach
View file

@ -1,20 +1,34 @@
#!/usr/bin/env python3 #!/bin/sh
# This Source Code Form is subject to the terms of the Mozilla Public # This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this # License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/. # file, You can obtain one at https://mozilla.org/MPL/2.0/.
# The beginning of this script is both valid shell and valid python, such that
# the script starts with the shell and is reexecuted with `uv run`. This
# ensures that the Python provided by the virtual environment (in the .venv
# directory) is used. If the virtual environment does not exist, `uv run` will
# still use the correct version of Python given in `.python-version` and
# python/mach_bootstrap.py will provision a new environment that will be used
# for the subsequent runs.
''':' && {
MACH_DIR=$(dirname "$0");
run_in_nix_if_needed() {
if { [ -f /etc/NIXOS ] || [ -n "${MACH_USE_NIX}" ]; } && [ -z "${IN_NIX_SHELL}" ]; then
EXTRA_NIX_ARGS=${SERVO_ANDROID_BUILD:+'--arg buildAndroid true'}
echo "NOTE: Entering nix-shell ${MACH_DIR}/shell.nix"
exec nix-shell "${MACH_DIR}/shell.nix" $EXTRA_NIX_ARGS --run "$*"
else
exec "$@"
fi
}
run_in_nix_if_needed uv run python ${MACH_DIR}/mach "$@"
}
'''
import os import os
import sys import sys
# Destructure because version_info > max_ver is true when running the same version.
ver = (sys.version_info[0], sys.version_info[1])
min_ver = (3, 10)
max_ver = (3, 12) # WPT does not support Python 3.13. See issue #34095.
if ver < min_ver or ver > max_ver:
print("mach does not support python {0}.{1}, please install 3.{2} <= python <= 3.{3}" \
.format(ver[0], ver[1], min_ver[1], max_ver[1]))
sys.exit(1)
def main(args): def main(args):
topdir = os.path.abspath(os.path.dirname(sys.argv[0])) topdir = os.path.abspath(os.path.dirname(sys.argv[0]))
sys.path.insert(0, os.path.join(topdir, "python")) sys.path.insert(0, os.path.join(topdir, "python"))
@ -27,20 +41,4 @@ def main(args):
if __name__ == '__main__': if __name__ == '__main__':
sys.dont_write_bytecode = True main(sys.argv)
need_nix_shell = os.path.exists('/etc/NIXOS') or 'MACH_USE_NIX' in os.environ
if need_nix_shell and not 'IN_NIX_SHELL' in os.environ:
import subprocess
from shlex import quote
mach_dir = os.path.abspath(os.path.dirname(__file__))
build_android_args = ['--arg', 'buildAndroid', 'true'] if 'SERVO_ANDROID_BUILD' in os.environ else []
print(f'NOTE: Entering nix-shell {mach_dir}/shell.nix')
try:
# sys argv already contains the ./mach part, so we just need to pass it as-is
result = subprocess.run(['nix-shell', f'{mach_dir}/shell.nix'] + build_android_args + ['--run', ' '.join(map(quote, sys.argv))])
sys.exit(result.returncode)
except KeyboardInterrupt:
sys.exit(0)
else:
main(sys.argv)

View file

@ -1,9 +1,4 @@
@echo off @echo off
set workdir=%~dp0
where /Q py.exe set workdir=%~dp0
IF %ERRORLEVEL% NEQ 0 ( uv run python %workdir%mach %*
python %workdir%mach %*
) ELSE (
py -3 %workdir%mach %*
)

View file

@ -4,10 +4,9 @@
import hashlib import hashlib
import os import os
import platform
import site
import subprocess import subprocess
import sys import sys
import runpy
SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
TOP_DIR = os.path.abspath(os.path.join(SCRIPT_PATH, "..")) TOP_DIR = os.path.abspath(os.path.join(SCRIPT_PATH, ".."))
@ -81,35 +80,16 @@ CATEGORIES = {
} }
# venv calls its scripts folder "bin" on non-Windows and "Scripts" on Windows. def _process_exec(args, cwd):
def _get_virtualenv_script_dir():
if os.name == "nt" and os.sep != "/":
return "Scripts"
return "bin"
# venv names its lib folder something like "lib/python3.11/site-packages" on
# non-Windows and "Lib\site-packages" on Window.
def _get_virtualenv_lib_dir():
if os.name == "nt" and os.sep != "/":
return os.path.join("Lib", "site-packages")
return os.path.join(
"lib",
f"python{sys.version_info[0]}.{sys.version_info[1]}",
"site-packages"
)
def _process_exec(args):
try: try:
subprocess.check_output(args, stderr=subprocess.STDOUT) subprocess.check_output(args, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as exception: except subprocess.CalledProcessError as exception:
print(exception.output.decode(sys.stdout.encoding)) print(exception.output.decode(sys.stdout.encoding))
print(f"Process failed with return code: {exception.returncode}") print(f"Process failed with return code: {exception.returncode}")
sys.exit(1) sys.exit(1)
def install_virtual_env_requirements(project_path: str, python: str, virtualenv_path: str): def install_virtual_env_requirements(project_path: str, virtualenv_path: str):
requirements_paths = [ requirements_paths = [
os.path.join(project_path, "python", "requirements.txt"), os.path.join(project_path, "python", "requirements.txt"),
os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",), os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",),
@ -131,45 +111,32 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_
requirements_hash = requirements_hasher.hexdigest() requirements_hash = requirements_hasher.hexdigest()
if marker_hash != requirements_hash: if marker_hash != requirements_hash:
print(" * Upgrading pip...")
_process_exec([python, "-m", "pip", "install", "--upgrade", "pip"])
print(" * Installing Python requirements...") print(" * Installing Python requirements...")
_process_exec([python, "-m", "pip", "install", "-I", pip_install_command = ["uv", "pip", "install"]
"-r", requirements_paths[0], for requirements in requirements_paths:
"-r", requirements_paths[1], pip_install_command.extend(["-r", requirements])
"-r", requirements_paths[2]]) _process_exec(pip_install_command, cwd=project_path)
with open(marker_path, "w") as marker_file: with open(marker_path, "w") as marker_file:
marker_file.write(requirements_hash) marker_file.write(requirements_hash)
def _activate_virtualenv(topdir): def _activate_virtualenv(topdir):
virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1])) virtualenv_path = os.path.join(topdir, ".venv")
python = sys.executable
if os.environ.get("VIRTUAL_ENV") != virtualenv_path: if os.environ.get("VIRTUAL_ENV") != virtualenv_path:
venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir())
if not os.path.exists(virtualenv_path): if not os.path.exists(virtualenv_path):
print(" * Setting up virtual environment...") print(" * Setting up virtual environment...")
_process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path]) _process_exec(["uv", "venv"], cwd=topdir)
# This general approach is taken from virtualenv's `activate_this.py`. script_dir = "Scripts" if _is_windows() else "bin"
os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)]) runpy.run_path(os.path.join(virtualenv_path, script_dir, 'activate_this.py'))
os.environ["VIRTUAL_ENV"] = virtualenv_path
prev_length = len(sys.path) install_virtual_env_requirements(topdir, virtualenv_path)
lib_path = os.path.realpath(os.path.join(virtualenv_path, _get_virtualenv_lib_dir()))
site.addsitedir(lib_path)
sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
sys.real_prefix = sys.prefix # Turn off warnings about deprecated syntax in our indirect dependencies.
sys.prefix = virtualenv_path # TODO: Find a better approach for doing this.
import warnings
# Use the python in our venv for subprocesses, not the python we were originally run with. warnings.filterwarnings('ignore', category=SyntaxWarning, module=r'.*.venv')
# Otherwise pip may still try to write to the wrong site-packages directory.
python = os.path.join(venv_script_path, "python")
install_virtual_env_requirements(topdir, python, virtualenv_path)
def _ensure_case_insensitive_if_windows(): def _ensure_case_insensitive_if_windows():
@ -221,13 +188,6 @@ def bootstrap(topdir):
print('Current path:', topdir) print('Current path:', topdir)
sys.exit(1) sys.exit(1)
# Ensure we are running Python 3.10+. We put this check here so we generate a
# user-friendly error message rather than a cryptic stack trace on module import.
if sys.version_info < (3, 10):
print('Python3 (>=3.10) is required to run mach.')
print('You are running Python', platform.python_version())
sys.exit(1)
_activate_virtualenv(topdir) _activate_virtualenv(topdir)
def populate_context(context, key=None): def populate_context(context, key=None):

View file

@ -175,7 +175,7 @@ class MachCommands(CommandBase):
return status return status
@Command('clean', @Command('clean',
description='Clean the target/ and python/_venv[version]/ directories', description='Clean the target/ and Python virtual environment directories',
category='build') category='build')
@CommandArgument('--manifest-path', @CommandArgument('--manifest-path',
default=None, default=None,
@ -188,8 +188,7 @@ class MachCommands(CommandBase):
def clean(self, manifest_path=None, params=[], verbose=False): def clean(self, manifest_path=None, params=[], verbose=False):
self.ensure_bootstrapped() self.ensure_bootstrapped()
virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1]) virtualenv_path = path.join(self.get_top_dir(), '.venv')
virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname)
if path.exists(virtualenv_path): if path.exists(virtualenv_path):
print('Removing virtualenv directory: %s' % virtualenv_path) print('Removing virtualenv directory: %s' % virtualenv_path)
shutil.rmtree(virtualenv_path) shutil.rmtree(virtualenv_path)

View file

@ -80,8 +80,7 @@ class PostBuildCommands(CommandBase):
@CommandBase.allow_target_configuration @CommandBase.allow_target_configuration
def run(self, servo_binary: str, params, debugger=False, debugger_cmd=None, def run(self, servo_binary: str, params, debugger=False, debugger_cmd=None,
headless=False, software=False, emulator=False, usb=False): headless=False, software=False, emulator=False, usb=False):
self._run(servo_binary, params, debugger, debugger_cmd, return self._run(servo_binary, params, debugger, debugger_cmd, headless, software, emulator, usb)
headless, software, emulator, usb)
def _run(self, servo_binary: str, params, debugger=False, debugger_cmd=None, def _run(self, servo_binary: str, params, debugger=False, debugger_cmd=None,
headless=False, software=False, emulator=False, usb=False): headless=False, software=False, emulator=False, usb=False):

View file

@ -18,7 +18,7 @@ import subprocess
import textwrap import textwrap
import json import json
from python.servo.post_build_commands import PostBuildCommands from servo.post_build_commands import PostBuildCommands
import wpt import wpt
import wpt.manifestupdate import wpt.manifestupdate
import wpt.run import wpt.run

View file

@ -41,7 +41,6 @@ directories = [
"./tests/wpt/mozilla/tests/mozilla/referrer-policy", "./tests/wpt/mozilla/tests/mozilla/referrer-policy",
"./tests/wpt/mozilla/tests/webgl", "./tests/wpt/mozilla/tests/webgl",
"./python/tidy/tests", "./python/tidy/tests",
"./python/_v*",
"./python/mach", "./python/mach",
# Generated and upstream code combined with our own. Could use cleanup # Generated and upstream code combined with our own. Could use cleanup
"./target", "./target",

View file

@ -5,7 +5,7 @@
buildAndroid ? false buildAndroid ? false
}: }:
with import (builtins.fetchTarball { with import (builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/d04953086551086b44b6f3c6b7eeb26294f207da.tar.gz"; url = "https://github.com/NixOS/nixpkgs/archive/5d67ea6b4b63378b9c13be21e2ec9d1afc921713.tar.gz";
}) { }) {
overlays = [ overlays = [
(import (builtins.fetchTarball { (import (builtins.fetchTarball {
@ -84,7 +84,13 @@ stdenv.mkDerivation (androidEnvironment // {
# Build utilities # Build utilities
cmake dbus gcc git pkg-config which llvm perl yasm m4 cmake dbus gcc git pkg-config which llvm perl yasm m4
(python3.withPackages (ps: with ps; [virtualenv pip dbus]))
# Ensure the Python version is same as the one in `.python-version` file so
# that `uv` will just symlink to the one in nix store. Otherwise `uv` will
# download a pre-built binary that won't work on nix.
# FIXME: dbus python module needs to be installed into the virtual environment.
python312
uv
# This pins gnumake to 4.3 since 4.4 breaks jobserver # This pins gnumake to 4.3 since 4.4 breaks jobserver
# functionality in mozjs and causes builds to be extremely # functionality in mozjs and causes builds to be extremely