Replace virtualenv with Python's built-in venv (#30377)

* Replace virtualenv with Python's built-in venv.

* Apply Delan's suggestions and make a couple small fixes

- Fix a tidy warning about directories that don't exist
- Use shutil instead of the redundant get_exec_path
- Miscellaneous cleanups

* Fix typo in environment variable

* fix bug where pip still tries to the wrong site-packages

---------

Co-authored-by: Martin Robinson <mrobinson@igalia.com>
Co-authored-by: Delan Azabani <dazabani@igalia.com>

.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
           fetch-depth: 2
       - name: Bootstrap
         run: |
-          python3 -m pip install --upgrade pip virtualenv
+          python3 -m pip install --upgrade pip
           sudo apt update
           python3 ./mach bootstrap
       - name: Compile docs

.github/workflows/linux-wpt.yml

@@ -57,7 +57,7 @@ jobs:
         run: tar -xzf release-binary/target.tar.gz
       - name: Prep test environment
         run: |
-          python3 -m pip install --upgrade pip virtualenv
+          python3 -m pip install --upgrade pip
           sudo apt update
           sudo apt install -qy --no-install-recommends libgl1 libssl1.1 libdbus-1-3 libxcb-xfixes0-dev libxcb-shape0-dev libunwind8 libgl1-mesa-dri mesa-vulkan-drivers libegl1-mesa
           sudo apt install ./libffi6_3.2.1-8_amd64.deb

.github/workflows/linux.yml

@@ -87,7 +87,7 @@ jobs:
           crate: taplo-cli
           locked: true
       - name: Bootstrap Python
-        run: python3 -m pip install --upgrade pip virtualenv
+        run: python3 -m pip install --upgrade pip
       - name: Bootstrap dependencies
         run: sudo apt update && python3 ./mach bootstrap
       - name: Tidy

.github/workflows/mac-wpt.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Prep test environment
         run: |
           gtar -xzf target.tar.gz
-          python3 -m pip install --upgrade pip virtualenv
+          python3 -m pip install --upgrade pip
           python3 ./mach bootstrap
       - name: Smoketest
         run: python3 ./mach smoketest --${{ inputs.profile }}

.github/workflows/mac.yml

@@ -82,7 +82,7 @@ jobs:
           locked: true
       - name: Bootstrap
         run: |
-          python3 -m pip install --upgrade pip virtualenv
+          python3 -m pip install --upgrade pip
           python3 ./mach bootstrap
           brew install gnu-tar
       - name: Build (${{ inputs.profile }})

.github/workflows/nightly-rust.yml

@@ -24,7 +24,7 @@ jobs:
         run: echo nightly > rust-toolchain
       - name: Bootstrap
         run: |
-          python3 -m pip install --upgrade pip virtualenv
+          python3 -m pip install --upgrade pip
           sudo apt update
           python3 ./mach bootstrap
       - name: Release build

.github/workflows/scheduled-wpt-import.yml

@@ -35,7 +35,7 @@ jobs:
       - uses: actions/download-artifact@v3
       - name: Prep environment
         run: |
-          python3 -m pip install --upgrade pip virtualenv
+          python3 -m pip install --upgrade pip
           sudo apt update
           python3 ./mach bootstrap
       - name: Add upstream remote

.github/workflows/windows.yml

@@ -78,7 +78,7 @@ jobs:
           # this point crown is not installed yet.
           RUSTC: "rustc"
         run: |
-          python -m pip install --upgrade pip virtualenv
+          python -m pip install --upgrade pip
           python mach fetch
           python mach bootstrap-gstreamer
           cargo install --path support/crown

.gitignore

@@ -12,6 +12,7 @@
 /ports/android/local.properties
 /ports/android/obj
 /python/_virtualenv*
+/python/_venv*
 /python/tidy/servo_tidy.egg-info
 /tests/wpt/sync
 *.pkl

README.md

@@ -25,18 +25,17 @@ manually, try the [manual build setup][manual-build].
 - Install [Xcode](https://developer.apple.com/xcode/)
 - Install [Homebrew](https://brew.sh/)
 - Run `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
-- Run `pip3 install virtualenv`
 - Run `./mach bootstrap`<br/>
   *Note: This will install the recommended version of GStreamer globally on your system.*
 
 ### Linux
 
 - Run `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
-- Install Python and virtualenv
-    - **Debian-like:** Run `sudo apt install python3-virtualenv python3-pip`
-    - **Fedora:** Run `sudo dnf install python3 python3-virtualenv python3-pip python3-devel`
-    - **Arch:** Run `sudo pacman -S --needed python python-virtualenv python-pip`
-    - **Gentoo:** Run `sudo emerge dev-python/virtualenv dev-python/pip`
+- Install Python
+    - **Debian-like:** Run `sudo apt install python3-pip`
+    - **Fedora:** Run `sudo dnf install python3 python3-pip python3-devel`
+    - **Arch:** Run `sudo pacman -S --needed python python-pip`
+    - **Gentoo:** Run `sudo emerge dev-python/pip`
 - Run `./mach bootstrap`
 
 ### Windows

etc/ci/performance/README.md

@@ -113,13 +113,3 @@ If you want to test the data submission code in `submit_to_perfherder.py` withou
 * `vagrant ssh`
   * `./manage.py create_credentials <username> <email> "description"`, the email has to match your logged in user. Remember to log-in through the Web UI once before you run this.
   * Setup your Treeherder client ID and secret as environment variables `TREEHERDER_CLIENT_ID` and `TREEHERDER_CLIENT_SECRET`
-
-# Troubleshooting
-
- If you saw this error message:
-
-```
-venv/bin/activate: line 8: _OLD_VIRTUAL_PATH: unbound variable
-```
-
-That means your `virtualenv` is too old, try run `pip install -U virtualenv` to upgrade (If you installed ubuntu's `python-virtualenv` package, uninstall it first then install it through `pip`)

python/mach_bootstrap.py

@@ -4,8 +4,9 @@
 
 import os
 import platform
-import sys
+import site
 import shutil
+import sys
 
 from subprocess import Popen
 from tempfile import TemporaryFile
@@ -80,27 +81,26 @@ CATEGORIES = {
     }
 }
 
-# Possible names of executables
-# NOTE: Windows Python doesn't provide versioned executables, so we must use
-# the plain names. On MSYS, we still use Windows Python.
-PYTHON_NAMES = ["python-2.7", "python2.7", "python2", "python"]
-
-
-def _get_exec_path(names, is_valid_path=lambda _path: True):
-    for name in names:
-        path = shutil.which(name)
-        if path and is_valid_path(path):
-            return path
-    return None
-
 
+# venv calls its scripts folder "bin" on non-Windows and "Scripts" on Windows.
 def _get_virtualenv_script_dir():
-    # Virtualenv calls its scripts folder "bin" on linux/OSX/MSYS64 but "Scripts" on Windows
     if os.name == "nt" and os.sep != "/":
         return "Scripts"
     return "bin"
 
 
+# venv names its lib folder something like "lib/python3.11/site-packages" on
+# non-Windows and "Lib\site-packages" on Window.
+def _get_virtualenv_lib_dir():
+    if os.name == "nt" and os.sep != "/":
+        return os.path.join("Lib", "site-packages")
+    return os.path.join(
+        "lib",
+        f"python{sys.version_info[0]}.{sys.version_info[1]}",
+        "site-packages"
+    )
+
+
 def _process_exec(args):
     with TemporaryFile() as out:
         with TemporaryFile() as err:
@@ -130,32 +130,29 @@ def _process_exec(args):
 
 
 def _activate_virtualenv(topdir):
-    virtualenv_path = os.path.join(topdir, "python", "_virtualenv%d.%d" % (sys.version_info[0], sys.version_info[1]))
-    python = sys.executable   # If there was no python, mach wouldn't have run at all!
-    if not python:
-        sys.exit('Failed to find python executable for starting virtualenv.')
+    virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1]))
+    python = sys.executable
 
-    script_dir = _get_virtualenv_script_dir()
-    activate_path = os.path.join(virtualenv_path, script_dir, "activate_this.py")
-    need_pip_upgrade = False
-    if not (os.path.exists(virtualenv_path) and os.path.exists(activate_path)):
-        import importlib
-        try:
-            importlib.import_module('virtualenv')
-        except ModuleNotFoundError:
-            sys.exit("Python virtualenv is not installed. Please install it prior to running mach.")
+    if os.environ.get("VIRTUAL_ENV") != virtualenv_path:
+        venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir())
+        if not os.path.exists(virtualenv_path):
+            _process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path])
 
-        _process_exec([python, "-m", "virtualenv", "-p", python, "--system-site-packages", virtualenv_path])
+        # This general approach is taken from virtualenv's `activate_this.py`.
+        os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)])
+        os.environ["VIRTUAL_ENV"] = virtualenv_path
 
-        # We want to upgrade pip when virtualenv created for the first time
-        need_pip_upgrade = True
+        prev_length = len(sys.path)
+        lib_path = os.path.realpath(os.path.join(virtualenv_path, _get_virtualenv_lib_dir()))
+        site.addsitedir(lib_path)
+        sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
 
-    exec(compile(open(activate_path).read(), activate_path, 'exec'), dict(__file__=activate_path))
+        sys.real_prefix = sys.prefix
+        sys.prefix = virtualenv_path
 
-    python = _get_exec_path(PYTHON_NAMES,
-                            is_valid_path=lambda path: path.startswith(virtualenv_path))
-    if not python:
-        sys.exit("Python executable in virtualenv failed to activate.")
+        # Use the python in our venv for subprocesses, not the python we were originally run with.
+        # Otherwise pip may still try to write to the wrong site-packages directory.
+        python = os.path.join(venv_script_path, "python")
 
     # TODO: Right now, we iteratively install all the requirements by invoking
     # `pip install` each time. If it were the case that there were conflicting
@@ -168,11 +165,6 @@ def _activate_virtualenv(topdir):
         os.path.join(WPT_RUNNER_PATH, "requirements.txt",),
     ]
 
-    if need_pip_upgrade:
-        # Upgrade pip when virtualenv is created to fix the issue
-        # https://github.com/servo/servo/issues/11074
-        _process_exec([python, "-m", "pip", "install", "-I", "-U", "pip"])
-
     for req_rel_path in requirements_paths:
         req_path = os.path.join(topdir, req_rel_path)
         marker_file = req_rel_path.replace(os.path.sep, '-')

python/servo/build_commands.py

@@ -252,7 +252,7 @@ class MachCommands(CommandBase):
                     subprocess.call(["perl", "-i", "-pe", expr, target_path])
 
     @Command('clean',
-             description='Clean the target/ and python/_virtualenv[version]/ directories',
+             description='Clean the target/ and python/_venv[version]/ directories',
              category='build')
     @CommandArgument('--manifest-path',
                      default=None,
@@ -265,7 +265,7 @@ class MachCommands(CommandBase):
     def clean(self, manifest_path=None, params=[], verbose=False):
         self.ensure_bootstrapped()
 
-        virtualenv_fname = '_virtualenv%d.%d' % (sys.version_info[0], sys.version_info[1])
+        virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1])
         virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname)
         if path.exists(virtualenv_path):
             print('Removing virtualenv directory: %s' % virtualenv_path)

python/servo/command_base.py

@@ -36,7 +36,6 @@ from xml.etree.ElementTree import XML
 
 import toml
 
-from mach_bootstrap import _get_exec_path
 from mach.decorators import CommandArgument, CommandArgumentGroup
 from mach.registrar import Registrar
 
@@ -618,8 +617,8 @@ class CommandBase(object):
             host_suffix = "x86_64"
         host = os_type + "-" + host_suffix
 
-        host_cc = env.get('HOST_CC') or _get_exec_path(["clang"]) or _get_exec_path(["gcc"])
-        host_cxx = env.get('HOST_CXX') or _get_exec_path(["clang++"]) or _get_exec_path(["g++"])
+        host_cc = env.get('HOST_CC') or shutil.which(["clang"]) or util.whichget_exec_path(["gcc"])
+        host_cxx = env.get('HOST_CXX') or util.whichget_exec_path(["clang++"]) or util.whichget_exec_path(["g++"])
 
         llvm_toolchain = path.join(env['ANDROID_NDK'], "toolchains", "llvm", "prebuilt", host)
         gcc_toolchain = path.join(env['ANDROID_NDK'], "toolchains",

servo-tidy.toml

@@ -105,7 +105,7 @@ directories = [
     "./tests/wpt/mozilla/tests/mozilla/referrer-policy",
     "./tests/wpt/mozilla/tests/webgl",
     "./python/tidy/tests",
-    "./python/_virtualenv*",
+    "./python/_v*",
     "./python/mach",
     # Generated and upstream code combined with our own. Could use cleanup
     "./target",