diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index b99c0ab8789..de8af871568 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -17,7 +17,7 @@ jobs:
fetch-depth: 2
- name: Bootstrap
run: |
- python3 -m pip install --upgrade pip virtualenv
+ python3 -m pip install --upgrade pip
sudo apt update
python3 ./mach bootstrap
- name: Compile docs
diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml
index f4ab2c139ab..9f18e80ba18 100644
--- a/.github/workflows/linux-wpt.yml
+++ b/.github/workflows/linux-wpt.yml
@@ -57,7 +57,7 @@ jobs:
run: tar -xzf release-binary/target.tar.gz
- name: Prep test environment
run: |
- python3 -m pip install --upgrade pip virtualenv
+ python3 -m pip install --upgrade pip
sudo apt update
sudo apt install -qy --no-install-recommends libgl1 libssl1.1 libdbus-1-3 libxcb-xfixes0-dev libxcb-shape0-dev libunwind8 libgl1-mesa-dri mesa-vulkan-drivers libegl1-mesa
sudo apt install ./libffi6_3.2.1-8_amd64.deb
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 6d6d72b6400..1bebe2e63a9 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -87,7 +87,7 @@ jobs:
crate: taplo-cli
locked: true
- name: Bootstrap Python
- run: python3 -m pip install --upgrade pip virtualenv
+ run: python3 -m pip install --upgrade pip
- name: Bootstrap dependencies
run: sudo apt update && python3 ./mach bootstrap
- name: Tidy
diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml
index efdd10ce2ad..ca19f60a3be 100644
--- a/.github/workflows/mac-wpt.yml
+++ b/.github/workflows/mac-wpt.yml
@@ -43,7 +43,7 @@ jobs:
- name: Prep test environment
run: |
gtar -xzf target.tar.gz
- python3 -m pip install --upgrade pip virtualenv
+ python3 -m pip install --upgrade pip
python3 ./mach bootstrap
- name: Smoketest
run: python3 ./mach smoketest --${{ inputs.profile }}
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
index 1b593810438..e7e0b2c75bd 100644
--- a/.github/workflows/mac.yml
+++ b/.github/workflows/mac.yml
@@ -82,7 +82,7 @@ jobs:
locked: true
- name: Bootstrap
run: |
- python3 -m pip install --upgrade pip virtualenv
+ python3 -m pip install --upgrade pip
python3 ./mach bootstrap
brew install gnu-tar
- name: Build (${{ inputs.profile }})
diff --git a/.github/workflows/nightly-rust.yml b/.github/workflows/nightly-rust.yml
index df31f969b51..10b04f3e9ed 100644
--- a/.github/workflows/nightly-rust.yml
+++ b/.github/workflows/nightly-rust.yml
@@ -24,7 +24,7 @@ jobs:
run: echo nightly > rust-toolchain
- name: Bootstrap
run: |
- python3 -m pip install --upgrade pip virtualenv
+ python3 -m pip install --upgrade pip
sudo apt update
python3 ./mach bootstrap
- name: Release build
diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml
index 9b2bea295f0..bc4166b05b0 100644
--- a/.github/workflows/scheduled-wpt-import.yml
+++ b/.github/workflows/scheduled-wpt-import.yml
@@ -35,7 +35,7 @@ jobs:
- uses: actions/download-artifact@v3
- name: Prep environment
run: |
- python3 -m pip install --upgrade pip virtualenv
+ python3 -m pip install --upgrade pip
sudo apt update
python3 ./mach bootstrap
- name: Add upstream remote
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index b8c18701c61..c2f76078f91 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -78,7 +78,7 @@ jobs:
# this point crown is not installed yet.
RUSTC: "rustc"
run: |
- python -m pip install --upgrade pip virtualenv
+ python -m pip install --upgrade pip
python mach fetch
python mach bootstrap-gstreamer
cargo install --path support/crown
diff --git a/.gitignore b/.gitignore
index 8622eb8484c..b9785f0beaf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
/ports/android/local.properties
/ports/android/obj
/python/_virtualenv*
+/python/_venv*
/python/tidy/servo_tidy.egg-info
/tests/wpt/sync
*.pkl
diff --git a/README.md b/README.md
index 9e6c7563663..025b713c64f 100644
--- a/README.md
+++ b/README.md
@@ -25,18 +25,17 @@ manually, try the [manual build setup][manual-build].
- Install [Xcode](https://developer.apple.com/xcode/)
- Install [Homebrew](https://brew.sh/)
- Run `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
-- Run `pip3 install virtualenv`
- Run `./mach bootstrap`
*Note: This will install the recommended version of GStreamer globally on your system.*
### Linux
- Run `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
-- Install Python and virtualenv
- - **Debian-like:** Run `sudo apt install python3-virtualenv python3-pip`
- - **Fedora:** Run `sudo dnf install python3 python3-virtualenv python3-pip python3-devel`
- - **Arch:** Run `sudo pacman -S --needed python python-virtualenv python-pip`
- - **Gentoo:** Run `sudo emerge dev-python/virtualenv dev-python/pip`
+- Install Python
+ - **Debian-like:** Run `sudo apt install python3-pip`
+ - **Fedora:** Run `sudo dnf install python3 python3-pip python3-devel`
+ - **Arch:** Run `sudo pacman -S --needed python python-pip`
+ - **Gentoo:** Run `sudo emerge dev-python/pip`
- Run `./mach bootstrap`
### Windows
diff --git a/etc/ci/performance/README.md b/etc/ci/performance/README.md
index 92c20ee5ec5..a269a710858 100644
--- a/etc/ci/performance/README.md
+++ b/etc/ci/performance/README.md
@@ -113,13 +113,3 @@ If you want to test the data submission code in `submit_to_perfherder.py` withou
* `vagrant ssh`
* `./manage.py create_credentials "description"`, the email has to match your logged in user. Remember to log-in through the Web UI once before you run this.
* Setup your Treeherder client ID and secret as environment variables `TREEHERDER_CLIENT_ID` and `TREEHERDER_CLIENT_SECRET`
-
-# Troubleshooting
-
- If you saw this error message:
-
-```
-venv/bin/activate: line 8: _OLD_VIRTUAL_PATH: unbound variable
-```
-
-That means your `virtualenv` is too old, try run `pip install -U virtualenv` to upgrade (If you installed ubuntu's `python-virtualenv` package, uninstall it first then install it through `pip`)
diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py
index c8085315219..53f7adb3fc4 100644
--- a/python/mach_bootstrap.py
+++ b/python/mach_bootstrap.py
@@ -4,8 +4,9 @@
import os
import platform
-import sys
+import site
import shutil
+import sys
from subprocess import Popen
from tempfile import TemporaryFile
@@ -80,27 +81,26 @@ CATEGORIES = {
}
}
-# Possible names of executables
-# NOTE: Windows Python doesn't provide versioned executables, so we must use
-# the plain names. On MSYS, we still use Windows Python.
-PYTHON_NAMES = ["python-2.7", "python2.7", "python2", "python"]
-
-
-def _get_exec_path(names, is_valid_path=lambda _path: True):
- for name in names:
- path = shutil.which(name)
- if path and is_valid_path(path):
- return path
- return None
-
+# venv calls its scripts folder "bin" on non-Windows and "Scripts" on Windows.
def _get_virtualenv_script_dir():
- # Virtualenv calls its scripts folder "bin" on linux/OSX/MSYS64 but "Scripts" on Windows
if os.name == "nt" and os.sep != "/":
return "Scripts"
return "bin"
+# venv names its lib folder something like "lib/python3.11/site-packages" on
+# non-Windows and "Lib\site-packages" on Window.
+def _get_virtualenv_lib_dir():
+ if os.name == "nt" and os.sep != "/":
+ return os.path.join("Lib", "site-packages")
+ return os.path.join(
+ "lib",
+ f"python{sys.version_info[0]}.{sys.version_info[1]}",
+ "site-packages"
+ )
+
+
def _process_exec(args):
with TemporaryFile() as out:
with TemporaryFile() as err:
@@ -130,32 +130,29 @@ def _process_exec(args):
def _activate_virtualenv(topdir):
- virtualenv_path = os.path.join(topdir, "python", "_virtualenv%d.%d" % (sys.version_info[0], sys.version_info[1]))
- python = sys.executable # If there was no python, mach wouldn't have run at all!
- if not python:
- sys.exit('Failed to find python executable for starting virtualenv.')
+ virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1]))
+ python = sys.executable
- script_dir = _get_virtualenv_script_dir()
- activate_path = os.path.join(virtualenv_path, script_dir, "activate_this.py")
- need_pip_upgrade = False
- if not (os.path.exists(virtualenv_path) and os.path.exists(activate_path)):
- import importlib
- try:
- importlib.import_module('virtualenv')
- except ModuleNotFoundError:
- sys.exit("Python virtualenv is not installed. Please install it prior to running mach.")
+ if os.environ.get("VIRTUAL_ENV") != virtualenv_path:
+ venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir())
+ if not os.path.exists(virtualenv_path):
+ _process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path])
- _process_exec([python, "-m", "virtualenv", "-p", python, "--system-site-packages", virtualenv_path])
+ # This general approach is taken from virtualenv's `activate_this.py`.
+ os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)])
+ os.environ["VIRTUAL_ENV"] = virtualenv_path
- # We want to upgrade pip when virtualenv created for the first time
- need_pip_upgrade = True
+ prev_length = len(sys.path)
+ lib_path = os.path.realpath(os.path.join(virtualenv_path, _get_virtualenv_lib_dir()))
+ site.addsitedir(lib_path)
+ sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
- exec(compile(open(activate_path).read(), activate_path, 'exec'), dict(__file__=activate_path))
+ sys.real_prefix = sys.prefix
+ sys.prefix = virtualenv_path
- python = _get_exec_path(PYTHON_NAMES,
- is_valid_path=lambda path: path.startswith(virtualenv_path))
- if not python:
- sys.exit("Python executable in virtualenv failed to activate.")
+ # Use the python in our venv for subprocesses, not the python we were originally run with.
+ # Otherwise pip may still try to write to the wrong site-packages directory.
+ python = os.path.join(venv_script_path, "python")
# TODO: Right now, we iteratively install all the requirements by invoking
# `pip install` each time. If it were the case that there were conflicting
@@ -168,11 +165,6 @@ def _activate_virtualenv(topdir):
os.path.join(WPT_RUNNER_PATH, "requirements.txt",),
]
- if need_pip_upgrade:
- # Upgrade pip when virtualenv is created to fix the issue
- # https://github.com/servo/servo/issues/11074
- _process_exec([python, "-m", "pip", "install", "-I", "-U", "pip"])
-
for req_rel_path in requirements_paths:
req_path = os.path.join(topdir, req_rel_path)
marker_file = req_rel_path.replace(os.path.sep, '-')
diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py
index cd977dfe91d..fae27e25b9e 100644
--- a/python/servo/build_commands.py
+++ b/python/servo/build_commands.py
@@ -252,7 +252,7 @@ class MachCommands(CommandBase):
subprocess.call(["perl", "-i", "-pe", expr, target_path])
@Command('clean',
- description='Clean the target/ and python/_virtualenv[version]/ directories',
+ description='Clean the target/ and python/_venv[version]/ directories',
category='build')
@CommandArgument('--manifest-path',
default=None,
@@ -265,7 +265,7 @@ class MachCommands(CommandBase):
def clean(self, manifest_path=None, params=[], verbose=False):
self.ensure_bootstrapped()
- virtualenv_fname = '_virtualenv%d.%d' % (sys.version_info[0], sys.version_info[1])
+ virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1])
virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname)
if path.exists(virtualenv_path):
print('Removing virtualenv directory: %s' % virtualenv_path)
diff --git a/python/servo/command_base.py b/python/servo/command_base.py
index daf9b68dd8e..af44102b313 100644
--- a/python/servo/command_base.py
+++ b/python/servo/command_base.py
@@ -36,7 +36,6 @@ from xml.etree.ElementTree import XML
import toml
-from mach_bootstrap import _get_exec_path
from mach.decorators import CommandArgument, CommandArgumentGroup
from mach.registrar import Registrar
@@ -618,8 +617,8 @@ class CommandBase(object):
host_suffix = "x86_64"
host = os_type + "-" + host_suffix
- host_cc = env.get('HOST_CC') or _get_exec_path(["clang"]) or _get_exec_path(["gcc"])
- host_cxx = env.get('HOST_CXX') or _get_exec_path(["clang++"]) or _get_exec_path(["g++"])
+ host_cc = env.get('HOST_CC') or shutil.which(["clang"]) or util.whichget_exec_path(["gcc"])
+ host_cxx = env.get('HOST_CXX') or util.whichget_exec_path(["clang++"]) or util.whichget_exec_path(["g++"])
llvm_toolchain = path.join(env['ANDROID_NDK'], "toolchains", "llvm", "prebuilt", host)
gcc_toolchain = path.join(env['ANDROID_NDK'], "toolchains",
diff --git a/servo-tidy.toml b/servo-tidy.toml
index 066d2ec6f7e..e1f933bb610 100644
--- a/servo-tidy.toml
+++ b/servo-tidy.toml
@@ -105,7 +105,7 @@ directories = [
"./tests/wpt/mozilla/tests/mozilla/referrer-policy",
"./tests/wpt/mozilla/tests/webgl",
"./python/tidy/tests",
- "./python/_virtualenv*",
+ "./python/_v*",
"./python/mach",
# Generated and upstream code combined with our own. Could use cleanup
"./target",