From 4103421ba5dc401817128661d759bb18b0aec8f1 Mon Sep 17 00:00:00 2001 From: Mukilan Thiyagarajan Date: Mon, 9 Dec 2024 20:22:06 +0530 Subject: [PATCH] mach: switch to `uv` for managing python venv (#34504) This patch switches servo to use `uv` for both installing a pinned Python version as well as installing the dependency packages using `uv`'s pip compatible interface. It also introduces a new 'composite' GitHub action to setup python in the different CI workflows. There is no support for externally managed python installations and virtual environments. These could be added in the future. Fixes #34095 Signed-off-by: Mukilan Thiyagarajan --- .github/actions/setup-python/action.yml | 20 +++++++++++ .github/workflows/android.yml | 4 +-- .github/workflows/docs.yml | 3 +- .github/workflows/lint.yml | 9 ++--- .github/workflows/linux-wpt.yml | 3 +- .github/workflows/linux.yml | 9 ++--- .github/workflows/mac-wpt.yml | 8 ++--- .github/workflows/mac.yml | 8 ++--- .github/workflows/main.yml | 7 ++-- .github/workflows/ohos.yml | 4 +-- .github/workflows/pull-request-wpt-export.yml | 10 ++++-- .github/workflows/scheduled-wpt-import.yml | 3 +- .github/workflows/try-label.yml | 7 ++-- .github/workflows/try.yml | 7 ++-- .github/workflows/windows.yml | 6 ++-- .python-version | 1 + README.md | 16 +++++---- mach | 7 ++-- python/mach_bootstrap.py | 34 ++++++++----------- python/servo/build_commands.py | 5 ++- servo-tidy.toml | 1 - shell.nix | 8 ++++- 22 files changed, 100 insertions(+), 80 deletions(-) create mode 100644 .github/actions/setup-python/action.yml create mode 100644 .python-version diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml new file mode 100644 index 00000000000..9889c517e6b --- /dev/null +++ b/.github/actions/setup-python/action.yml @@ -0,0 +1,20 @@ +name: Setup Python and uv +inputs: + skip-python-setup: + required: false + description: "Whether to skip installing python using Github's `setup-python` action" + default: false +runs: + using: "composite" + steps: + # Use the setup-python action to take advantage of the cache. uv will + # symlink to this version. + - name: Setup system python + if: ${{ inputs.skip-python-setup != 'true' }} + uses: actions/setup-python@v5 + with: + python-version-file: '.python-version' + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.5.6" diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 6077daeedcf..5ce7fad25ec 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -59,8 +59,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip virtualenv + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: sudo apt update && python3 ./mach bootstrap --skip-lints - name: Set up JDK 17 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f546469f962..74530c40e1f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,9 +13,10 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap run: | - python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap --skip-lints - name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1d900e1c7bb..4527200526a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,9 +31,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - - uses: actions/setup-python@v5 - with: - python-version: '3.10' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Install taplo uses: baptiste0928/cargo-install@v3 with: @@ -46,8 +45,6 @@ jobs: locked: true # 0.16.2 requires Rust 1.81 or newer. version: '0.16.1' - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip - name: Bootstrap dependencies run: | sudo apt update @@ -57,4 +54,4 @@ jobs: run: | python3 ./mach clippy --use-crown --locked -- -- --deny warnings - name: Tidy - run: python3 ./mach test-tidy --no-progress --all \ No newline at end of file + run: python3 ./mach test-tidy --no-progress --all diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index 34857bdc9d0..f710518b462 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -52,9 +52,10 @@ jobs: path: ${{ inputs.profile }}-binary-linux - name: unPackage binary run: tar -xzf ${{ inputs.profile }}-binary-linux/target.tar.gz + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: | - python3 -m pip install --upgrade pip sudo apt update sudo apt install -qy --no-install-recommends mesa-vulkan-drivers python3 ./mach bootstrap --skip-lints diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index cfced0395c5..c2db7cfd17a 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -139,13 +139,10 @@ jobs: - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - - uses: actions/setup-python@v5 - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} + - name: Setup Python + uses: ./.github/actions/setup-python with: - python-version: '3.10' - - name: Bootstrap Python - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} - run: python3 -m pip install --upgrade pip + skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} - name: Bootstrap dependencies if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: | diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml index 74a1e1439fe..610da047a5f 100644 --- a/.github/workflows/mac-wpt.yml +++ b/.github/workflows/mac-wpt.yml @@ -41,15 +41,11 @@ jobs: - uses: actions/download-artifact@v4 with: name: ${{ inputs.profile }}-binary-macos - # Python 3.13 breaks wptrunner, so pin the version until - # web-platform-tests/wpt#48585 is fixed. - - uses: actions/setup-python@v5 - with: - python-version: '3.12' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Prep test environment run: | gtar -xzf target.tar.gz - python3 -m pip install --upgrade pip python3 ./mach bootstrap --skip-lints - name: Smoketest run: python3 ./mach smoketest --${{ inputs.profile }} diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 4467b564426..f0edacb9fb8 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -77,18 +77,14 @@ jobs: if: github.event_name == 'pull_request_target' with: ref: ${{ github.event.pull_request.head.sha }} - # Python 3.13 breaks wptrunner, so pin the version until - # web-platform-tests/wpt#48585 is fixed. - - uses: actions/setup-python@v5 - with: - python-version: '3.12' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - name: Bootstrap run: | - python3 -m pip install --upgrade pip python3 ./mach bootstrap --skip-lints brew install gnu-tar - name: Build (${{ inputs.profile }}) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 83e09cd3977..2d043312f0a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,15 +20,16 @@ jobs: outputs: configuration: ${{ steps.configuration.outputs.result }} steps: - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: fetch-depth: 1 sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Get Configuration id: configuration run: | diff --git a/.github/workflows/ohos.yml b/.github/workflows/ohos.yml index b60f7f3f121..2690ea100b8 100644 --- a/.github/workflows/ohos.yml +++ b/.github/workflows/ohos.yml @@ -53,8 +53,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip virtualenv + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: sudo apt update && python3 ./mach bootstrap --skip-lints - name: Setup OpenHarmony SDK diff --git a/.github/workflows/pull-request-wpt-export.yml b/.github/workflows/pull-request-wpt-export.yml index 2942fe9f2cd..1c3ee1a6cf0 100644 --- a/.github/workflows/pull-request-wpt-export.yml +++ b/.github/workflows/pull-request-wpt-export.yml @@ -28,10 +28,16 @@ jobs: # using the token specified here. # See https://github.com/actions/checkout/issues/162. token: ${{ secrets.WPT_SYNC_TOKEN }} + - name: Setup Python + uses: ./.github/actions/setup-python - name: Install requirements - run: pip install -r servo/python/requirements.txt + run: | + uv venv + uv pip install -r servo/python/requirements.txt - name: Process pull request - run: servo/python/wpt/export.py + run: | + source .venv/bin/activate + servo/python/wpt/export.py env: GITHUB_CONTEXT: ${{ toJson(github) }} WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }} diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml index 1ed65b12c32..145133b817c 100644 --- a/.github/workflows/scheduled-wpt-import.yml +++ b/.github/workflows/scheduled-wpt-import.yml @@ -35,9 +35,10 @@ jobs: - uses: actions/download-artifact@v4 with: name: wpt-full-logs-linux-layout-2020 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Prep environment run: | - python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap - name: Add upstream remote diff --git a/.github/workflows/try-label.yml b/.github/workflows/try-label.yml index 610aeb883c9..1a95432cd7d 100644 --- a/.github/workflows/try-label.yml +++ b/.github/workflows/try-label.yml @@ -77,14 +77,15 @@ jobs: } return try_string; - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Parse Labels if: ${{ steps.try_string.outputs.result }} id: configuration diff --git a/.github/workflows/try.yml b/.github/workflows/try.yml index ca81a8f91cd..d81715fe92d 100644 --- a/.github/workflows/try.yml +++ b/.github/workflows/try.yml @@ -32,15 +32,16 @@ jobs: outputs: configuration: ${{ steps.configuration.outputs.result }} steps: - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: fetch-depth: 1 sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Get Full Configuration id: full_config run: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a59c23dc919..a9c0805c80c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -114,13 +114,13 @@ jobs: choco install wixtoolset echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - uses: actions/setup-python@v5 - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} + - name: Setup Python + uses: ./.github/actions/setup-python with: - python-version: "3.10" + skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} - name: Bootstrap if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: | - python -m pip install --upgrade pip python mach fetch python mach bootstrap-gstreamer # For some reason WiX isn't currently on the GitHub runner path. This is a diff --git a/.python-version b/.python-version new file mode 100644 index 00000000000..e4fba218358 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/README.md b/README.md index 0c83a837d29..e9ea754bd6a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ For more detailed build instructions, see the Servo book under [Setting up your ### macOS -- Download and install [`python`](https://www.python.org/downloads/macos/) (version 3.10 to 3.12), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/). +- Download and install [`python`](https://www.python.org/downloads/macos/), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/). +- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Restart your shell to make sure `cargo` is available - Install the other dependencies: `./mach bootstrap` @@ -25,11 +26,12 @@ For more detailed build instructions, see the Servo book under [Setting up your ### Linux -- Install `curl` and `python` (version 3.10 to 3.12): - - Arch: `sudo pacman -S --needed curl python python-pip` - - Debian, Ubuntu: `sudo apt install curl python3-pip python3-venv python3-setuptools` - - Fedora: `sudo dnf install curl python3 python3-pip python3-devel` - - Gentoo: `sudo emerge net-misc/curl dev-python/pip` +- Install `curl` and `python`: + - Arch: `sudo pacman -S --needed curl python` + - Debian, Ubuntu: `sudo apt install curl` + - Fedora: `sudo dnf install curl python3` + - Gentoo: `sudo emerge net-misc/curl` +- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Restart your shell to make sure `cargo` is available - Install the other dependencies: `./mach bootstrap` @@ -37,7 +39,7 @@ For more detailed build instructions, see the Servo book under [Setting up your ### Windows -- Download and install [`python`](https://www.python.org/downloads/windows/) (version 3.10 to 3.12), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/) +- Download and install [`python`](https://www.python.org/downloads/windows/), [`uv`](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/) - Be sure to select *Quick install via the Visual Studio Community installer* - In the Visual Studio Installer, ensure the following components are installed: - **Windows 10 SDK (10.0.19041.0)** (`Microsoft.VisualStudio.Component.Windows10SDK.19041`) diff --git a/mach b/mach index aa21bcef4f0..681bdbf4633 100755 --- a/mach +++ b/mach @@ -9,10 +9,9 @@ import sys # Destructure because version_info > max_ver is true when running the same version. ver = (sys.version_info[0], sys.version_info[1]) min_ver = (3, 10) -max_ver = (3, 12) # WPT does not support Python 3.13. See issue #34095. -if ver < min_ver or ver > max_ver: - print("mach does not support python {0}.{1}, please install 3.{2} <= python <= 3.{3}" \ - .format(ver[0], ver[1], min_ver[1], max_ver[1])) +if ver < min_ver: + print("mach requires at least version 3.{0} of Python. The version of Python installed in this system is {1}.{2}" \ + .format(min_ver[1], ver[0], ver[1])) sys.exit(1) def main(args): diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 43690c3d951..fc6e916ff80 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -4,7 +4,6 @@ import hashlib import os -import platform import site import subprocess import sys @@ -93,11 +92,13 @@ def _get_virtualenv_script_dir(): def _get_virtualenv_lib_dir(): if os.name == "nt" and os.sep != "/": return os.path.join("Lib", "site-packages") - return os.path.join( - "lib", - f"python{sys.version_info[0]}.{sys.version_info[1]}", - "site-packages" - ) + with open(".python-version", "r") as python_version_file: + python_version = python_version_file.read().strip() + return os.path.join( + "lib", + f"python{python_version}", + "site-packages" + ) def _process_exec(args): @@ -131,11 +132,8 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_ requirements_hash = requirements_hasher.hexdigest() if marker_hash != requirements_hash: - print(" * Upgrading pip...") - _process_exec([python, "-m", "pip", "install", "--upgrade", "pip"]) - print(" * Installing Python requirements...") - _process_exec([python, "-m", "pip", "install", "-I", + _process_exec(["uv", "pip", "install", "-r", requirements_paths[0], "-r", requirements_paths[1], "-r", requirements_paths[2]]) @@ -144,14 +142,14 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_ def _activate_virtualenv(topdir): - virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1])) + virtualenv_path = os.path.join(topdir, ".venv") python = sys.executable if os.environ.get("VIRTUAL_ENV") != virtualenv_path: venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir()) if not os.path.exists(virtualenv_path): print(" * Setting up virtual environment...") - _process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path]) + _process_exec(["uv", "venv"]) # This general approach is taken from virtualenv's `activate_this.py`. os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)]) @@ -171,6 +169,11 @@ def _activate_virtualenv(topdir): install_virtual_env_requirements(topdir, python, virtualenv_path) + # Turn off warnings about deprecated syntax in our indirect dependencies. + # TODO: Find a better approach for doing this. + import warnings + warnings.filterwarnings('ignore', category=SyntaxWarning, module=r'.*.venv') + def _ensure_case_insensitive_if_windows(): # The folder is called 'python'. By deliberately checking for it with the wrong case, we determine if the file @@ -221,13 +224,6 @@ def bootstrap(topdir): print('Current path:', topdir) sys.exit(1) - # Ensure we are running Python 3.10+. We put this check here so we generate a - # user-friendly error message rather than a cryptic stack trace on module import. - if sys.version_info < (3, 10): - print('Python3 (>=3.10) is required to run mach.') - print('You are running Python', platform.python_version()) - sys.exit(1) - _activate_virtualenv(topdir) def populate_context(context, key=None): diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index c48999f99f9..b21f89980c0 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -175,7 +175,7 @@ class MachCommands(CommandBase): return status @Command('clean', - description='Clean the target/ and python/_venv[version]/ directories', + description='Clean the target/ and Python virtual environment directories', category='build') @CommandArgument('--manifest-path', default=None, @@ -188,8 +188,7 @@ class MachCommands(CommandBase): def clean(self, manifest_path=None, params=[], verbose=False): self.ensure_bootstrapped() - virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1]) - virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname) + virtualenv_path = path.join(self.get_top_dir(), '.venv') if path.exists(virtualenv_path): print('Removing virtualenv directory: %s' % virtualenv_path) shutil.rmtree(virtualenv_path) diff --git a/servo-tidy.toml b/servo-tidy.toml index 64d33117a6a..7460f4092ce 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -117,7 +117,6 @@ directories = [ "./tests/wpt/mozilla/tests/mozilla/referrer-policy", "./tests/wpt/mozilla/tests/webgl", "./python/tidy/tests", - "./python/_v*", "./python/mach", # Generated and upstream code combined with our own. Could use cleanup "./target", diff --git a/shell.nix b/shell.nix index 2576e0e454f..fd110442cd6 100644 --- a/shell.nix +++ b/shell.nix @@ -84,7 +84,13 @@ stdenv.mkDerivation (androidEnvironment // { # Build utilities cmake dbus gcc git pkg-config which llvm perl yasm m4 - (python3.withPackages (ps: with ps; [virtualenv pip dbus])) + + # Ensure the Python version is same as the one in `.python-version` file so + # that `uv` will just symlink to the one in nix store. Otherwise `uv` will + # download a pre-built binary that won't work on nix. + # FIXME: dbus python module needs to be installed into the virtual environment. + python312 + uv # This pins gnumake to 4.3 since 4.4 breaks jobserver # functionality in mozjs and causes builds to be extremely