mach: switch to uv for managing python venv (#34504)

This patch switches servo to use `uv` for both installing a pinned
Python version as well as installing the dependency packages using
`uv`'s pip compatible interface. It also introduces a new 'composite'
GitHub action to setup python in the different CI workflows.

There is no support for externally managed python installations and
virtual environments. These could be added in the future.

Fixes #34095

Signed-off-by: Mukilan Thiyagarajan <mukilan@igalia.com>

.github/actions/setup-python/action.yml

@@ -0,0 +1,20 @@
+name: Setup Python and uv
+inputs:
+  skip-python-setup:
+    required: false
+    description: "Whether to skip installing python using Github's `setup-python` action"
+    default: false
+runs:
+  using: "composite"
+  steps:
+    # Use the setup-python action to take advantage of the cache. uv will
+    # symlink to this version.
+    - name: Setup system python
+      if: ${{ inputs.skip-python-setup != 'true' }}
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: '.python-version'
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "0.5.6"

.github/workflows/android.yml

@@ -59,8 +59,8 @@ jobs:
         uses: mozilla-actions/sccache-action@v0.0.6
       - name: Install crown
         run: cargo install --path support/crown
-      - name: Bootstrap Python
-        run: python3 -m pip install --upgrade pip virtualenv
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Bootstrap dependencies
         run: sudo apt update && python3 ./mach bootstrap --skip-lints
       - name: Set up JDK 17

.github/workflows/docs.yml

@@ -13,9 +13,10 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Bootstrap
         run: |
-          python3 -m pip install --upgrade pip
           sudo apt update
           python3 ./mach bootstrap --skip-lints
       - name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle.

.github/workflows/lint.yml

@@ -31,9 +31,8 @@ jobs:
         uses: mozilla-actions/sccache-action@v0.0.6
       - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle
         run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Install taplo
         uses: baptiste0928/cargo-install@v3
         with:
@@ -46,8 +45,6 @@ jobs:
           locked: true
           # 0.16.2 requires Rust 1.81 or newer.
           version: '0.16.1'
-      - name: Bootstrap Python
-        run: python3 -m pip install --upgrade pip
       - name: Bootstrap dependencies
         run: |
           sudo apt update
@@ -57,4 +54,4 @@ jobs:
         run: |
           python3 ./mach clippy --use-crown --locked -- -- --deny warnings
       - name: Tidy
-        run: python3 ./mach test-tidy --no-progress --all
+        run: python3 ./mach test-tidy --no-progress --all

.github/workflows/linux-wpt.yml

@@ -52,9 +52,10 @@ jobs:
           path: ${{ inputs.profile }}-binary-linux
       - name: unPackage binary
         run: tar -xzf ${{ inputs.profile }}-binary-linux/target.tar.gz
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Bootstrap dependencies
         run: |
-          python3 -m pip install --upgrade pip
           sudo apt update
           sudo apt install -qy --no-install-recommends mesa-vulkan-drivers
           python3 ./mach bootstrap --skip-lints

.github/workflows/linux.yml

@@ -139,13 +139,10 @@ jobs:
       - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle
         if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly
         run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
-      - uses: actions/setup-python@v5
-        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
         with:
-          python-version: '3.10'
-      - name: Bootstrap Python
-        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
-        run: python3 -m pip install --upgrade pip
+          skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
       - name: Bootstrap dependencies
         if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
         run: |

.github/workflows/mac-wpt.yml

@@ -41,15 +41,11 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: ${{ inputs.profile }}-binary-macos
-      # Python 3.13 breaks wptrunner, so pin the version until
-      # web-platform-tests/wpt#48585 is fixed.
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Prep test environment
         run: |
           gtar -xzf target.tar.gz
-          python3 -m pip install --upgrade pip
           python3 ./mach bootstrap --skip-lints
       - name: Smoketest
         run: python3 ./mach smoketest --${{ inputs.profile }}

.github/workflows/mac.yml

@@ -77,18 +77,14 @@ jobs:
         if: github.event_name == 'pull_request_target'
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      # Python 3.13 breaks wptrunner, so pin the version until
-      # web-platform-tests/wpt#48585 is fixed.
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.6
       - name: Install crown
         run: cargo install --path support/crown
       - name: Bootstrap
         run: |
-          python3 -m pip install --upgrade pip
           python3 ./mach bootstrap --skip-lints
           brew install gnu-tar
       - name: Build (${{ inputs.profile }})

.github/workflows/main.yml

@@ -20,15 +20,16 @@ jobs:
     outputs:
       configuration: ${{ steps.configuration.outputs.result }}
     steps:
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
           sparse-checkout: |
             python/servo/try_parser.py
+            .github/actions/setup-python
+            .python-version
           sparse-checkout-cone-mode: false
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Get Configuration
         id: configuration
         run: |

.github/workflows/ohos.yml

@@ -53,8 +53,8 @@ jobs:
         uses: mozilla-actions/sccache-action@v0.0.6
       - name: Install crown
         run: cargo install --path support/crown
-      - name: Bootstrap Python
-        run: python3 -m pip install --upgrade pip virtualenv
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Bootstrap dependencies
         run: sudo apt update && python3 ./mach bootstrap --skip-lints
       - name: Setup OpenHarmony SDK

.github/workflows/pull-request-wpt-export.yml

@@ -28,10 +28,16 @@ jobs:
           # using the token specified here.
           # See https://github.com/actions/checkout/issues/162.
           token: ${{ secrets.WPT_SYNC_TOKEN }}
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Install requirements
-        run: pip install -r servo/python/requirements.txt
+        run: |
+          uv venv
+          uv pip install -r servo/python/requirements.txt
       - name: Process pull request
-        run: servo/python/wpt/export.py
+        run: |
+         source .venv/bin/activate
+         servo/python/wpt/export.py
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
           WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }}

.github/workflows/scheduled-wpt-import.yml

@@ -35,9 +35,10 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: wpt-full-logs-linux-layout-2020
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Prep environment
         run: |
-          python3 -m pip install --upgrade pip
           sudo apt update
           python3 ./mach bootstrap
       - name: Add upstream remote

.github/workflows/try-label.yml

@@ -77,14 +77,15 @@ jobs:
             }
 
             return try_string;
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
       - uses: actions/checkout@v4
         with:
           sparse-checkout: |
             python/servo/try_parser.py
+            .github/actions/setup-python
+            .python-version
           sparse-checkout-cone-mode: false
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Parse Labels
         if: ${{ steps.try_string.outputs.result }}
         id: configuration

.github/workflows/try.yml

@@ -32,15 +32,16 @@ jobs:
     outputs:
       configuration: ${{ steps.configuration.outputs.result }}
     steps:
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
           sparse-checkout: |
             python/servo/try_parser.py
+            .github/actions/setup-python
+            .python-version
           sparse-checkout-cone-mode: false
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Get Full Configuration
         id: full_config
         run: |

.github/workflows/windows.yml

@@ -114,13 +114,13 @@ jobs:
           choco install wixtoolset
           echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
       - uses: actions/setup-python@v5
-        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
         with:
-          python-version: "3.10"
+          skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
       - name: Bootstrap
         if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
         run: |
-          python -m pip install --upgrade pip
           python mach fetch
           python mach bootstrap-gstreamer
       # For some reason WiX isn't currently on the GitHub runner path. This is a

.python-version

@@ -0,0 +1 @@
+3.12

README.md

@@ -17,7 +17,8 @@ For more detailed build instructions, see the Servo book under [Setting up your
 
 ### macOS
 
-- Download and install [`python`](https://www.python.org/downloads/macos/) (version 3.10 to 3.12), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/).
+- Download and install [`python`](https://www.python.org/downloads/macos/), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/).
+- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` 
 - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
 - Restart your shell to make sure `cargo` is available
 - Install the other dependencies: `./mach bootstrap`
@@ -25,11 +26,12 @@ For more detailed build instructions, see the Servo book under [Setting up your
 
 ### Linux
 
-- Install `curl` and `python` (version 3.10 to 3.12):
-  - Arch: `sudo pacman -S --needed curl python python-pip`
-  - Debian, Ubuntu: `sudo apt install curl python3-pip python3-venv python3-setuptools`
-  - Fedora: `sudo dnf install curl python3 python3-pip python3-devel`
-  - Gentoo: `sudo emerge net-misc/curl dev-python/pip`
+- Install `curl` and `python`:
+  - Arch: `sudo pacman -S --needed curl python`
+  - Debian, Ubuntu: `sudo apt install curl`
+  - Fedora: `sudo dnf install curl python3`
+  - Gentoo: `sudo emerge net-misc/curl`
+- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` 
 - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
 - Restart your shell to make sure `cargo` is available
 - Install the other dependencies: `./mach bootstrap`
@@ -37,7 +39,7 @@ For more detailed build instructions, see the Servo book under [Setting up your
 
 ### Windows
 
-- Download and install [`python`](https://www.python.org/downloads/windows/) (version 3.10 to 3.12), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/)
+- Download and install [`python`](https://www.python.org/downloads/windows/), [`uv`](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/)
   - Be sure to select *Quick install via the Visual Studio Community installer*
 - In the Visual Studio Installer, ensure the following components are installed:
   - **Windows 10 SDK (10.0.19041.0)** (`Microsoft.VisualStudio.Component.Windows10SDK.19041`)

mach

@@ -9,10 +9,9 @@ import sys
 # Destructure because version_info > max_ver is true when running the same version.
 ver = (sys.version_info[0], sys.version_info[1])
 min_ver = (3, 10)
-max_ver = (3, 12) # WPT does not support Python 3.13. See issue #34095.
-if ver < min_ver or ver > max_ver:
-    print("mach does not support python {0}.{1}, please install 3.{2} <= python <= 3.{3}" \
-        .format(ver[0], ver[1], min_ver[1], max_ver[1]))
+if ver < min_ver:
+    print("mach requires at least version 3.{0} of Python. The version of Python installed in this system is {1}.{2}" \
+        .format(min_ver[1], ver[0], ver[1]))
     sys.exit(1)
 
 def main(args):

python/mach_bootstrap.py

@@ -4,7 +4,6 @@
 
 import hashlib
 import os
-import platform
 import site
 import subprocess
 import sys
@@ -93,11 +92,13 @@ def _get_virtualenv_script_dir():
 def _get_virtualenv_lib_dir():
     if os.name == "nt" and os.sep != "/":
         return os.path.join("Lib", "site-packages")
-    return os.path.join(
-        "lib",
-        f"python{sys.version_info[0]}.{sys.version_info[1]}",
-        "site-packages"
-    )
+    with open(".python-version", "r") as python_version_file:
+        python_version = python_version_file.read().strip()
+        return os.path.join(
+            "lib",
+            f"python{python_version}",
+            "site-packages"
+        )
 
 
 def _process_exec(args):
@@ -131,11 +132,8 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_
     requirements_hash = requirements_hasher.hexdigest()
 
     if marker_hash != requirements_hash:
-        print(" * Upgrading pip...")
-        _process_exec([python, "-m", "pip", "install", "--upgrade", "pip"])
-
         print(" * Installing Python requirements...")
-        _process_exec([python, "-m", "pip", "install", "-I",
+        _process_exec(["uv", "pip", "install",
                        "-r", requirements_paths[0],
                        "-r", requirements_paths[1],
                        "-r", requirements_paths[2]])
@@ -144,14 +142,14 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_
 
 
 def _activate_virtualenv(topdir):
-    virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1]))
+    virtualenv_path = os.path.join(topdir, ".venv")
     python = sys.executable
 
     if os.environ.get("VIRTUAL_ENV") != virtualenv_path:
         venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir())
         if not os.path.exists(virtualenv_path):
             print(" * Setting up virtual environment...")
-            _process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path])
+            _process_exec(["uv", "venv"])
 
         # This general approach is taken from virtualenv's `activate_this.py`.
         os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)])
@@ -171,6 +169,11 @@ def _activate_virtualenv(topdir):
 
     install_virtual_env_requirements(topdir, python, virtualenv_path)
 
+    # Turn off warnings about deprecated syntax in our indirect dependencies.
+    # TODO: Find a better approach for doing this.
+    import warnings
+    warnings.filterwarnings('ignore', category=SyntaxWarning, module=r'.*.venv')
+
 
 def _ensure_case_insensitive_if_windows():
     # The folder is called 'python'. By deliberately checking for it with the wrong case, we determine if the file
@@ -221,13 +224,6 @@ def bootstrap(topdir):
         print('Current path:', topdir)
         sys.exit(1)
 
-    # Ensure we are running Python 3.10+. We put this check here so we generate a
-    # user-friendly error message rather than a cryptic stack trace on module import.
-    if sys.version_info < (3, 10):
-        print('Python3 (>=3.10) is required to run mach.')
-        print('You are running Python', platform.python_version())
-        sys.exit(1)
-
     _activate_virtualenv(topdir)
 
     def populate_context(context, key=None):

python/servo/build_commands.py

@@ -175,7 +175,7 @@ class MachCommands(CommandBase):
         return status
 
     @Command('clean',
-             description='Clean the target/ and python/_venv[version]/ directories',
+             description='Clean the target/ and Python virtual environment directories',
              category='build')
     @CommandArgument('--manifest-path',
                      default=None,
@@ -188,8 +188,7 @@ class MachCommands(CommandBase):
     def clean(self, manifest_path=None, params=[], verbose=False):
         self.ensure_bootstrapped()
 
-        virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1])
-        virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname)
+        virtualenv_path = path.join(self.get_top_dir(), '.venv')
         if path.exists(virtualenv_path):
             print('Removing virtualenv directory: %s' % virtualenv_path)
             shutil.rmtree(virtualenv_path)

servo-tidy.toml

@@ -117,7 +117,6 @@ directories = [
     "./tests/wpt/mozilla/tests/mozilla/referrer-policy",
     "./tests/wpt/mozilla/tests/webgl",
     "./python/tidy/tests",
-    "./python/_v*",
     "./python/mach",
     # Generated and upstream code combined with our own. Could use cleanup
     "./target",

shell.nix

@@ -84,7 +84,13 @@ stdenv.mkDerivation (androidEnvironment // {
 
     # Build utilities
     cmake dbus gcc git pkg-config which llvm perl yasm m4
-    (python3.withPackages (ps: with ps; [virtualenv pip dbus]))
+
+    # Ensure the Python version is same as the one in `.python-version` file so
+    # that `uv` will just symlink to the one in nix store. Otherwise `uv` will
+    # download a pre-built binary that won't work on nix.
+    # FIXME: dbus python module needs to be installed into the virtual environment.
+    python312
+    uv
 
     # This pins gnumake to 4.3 since 4.4 breaks jobserver
     # functionality in mozjs and causes builds to be extremely