diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b99c0ab8789..de8af871568 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 2 - name: Bootstrap run: | - python3 -m pip install --upgrade pip virtualenv + python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap - name: Compile docs diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index f4ab2c139ab..9f18e80ba18 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -57,7 +57,7 @@ jobs: run: tar -xzf release-binary/target.tar.gz - name: Prep test environment run: | - python3 -m pip install --upgrade pip virtualenv + python3 -m pip install --upgrade pip sudo apt update sudo apt install -qy --no-install-recommends libgl1 libssl1.1 libdbus-1-3 libxcb-xfixes0-dev libxcb-shape0-dev libunwind8 libgl1-mesa-dri mesa-vulkan-drivers libegl1-mesa sudo apt install ./libffi6_3.2.1-8_amd64.deb diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 6d6d72b6400..1bebe2e63a9 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -87,7 +87,7 @@ jobs: crate: taplo-cli locked: true - name: Bootstrap Python - run: python3 -m pip install --upgrade pip virtualenv + run: python3 -m pip install --upgrade pip - name: Bootstrap dependencies run: sudo apt update && python3 ./mach bootstrap - name: Tidy diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml index efdd10ce2ad..ca19f60a3be 100644 --- a/.github/workflows/mac-wpt.yml +++ b/.github/workflows/mac-wpt.yml @@ -43,7 +43,7 @@ jobs: - name: Prep test environment run: | gtar -xzf target.tar.gz - python3 -m pip install --upgrade pip virtualenv + python3 -m pip install --upgrade pip python3 ./mach bootstrap - name: Smoketest run: python3 ./mach smoketest --${{ inputs.profile }} diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 1b593810438..e7e0b2c75bd 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -82,7 +82,7 @@ jobs: locked: true - name: Bootstrap run: | - python3 -m pip install --upgrade pip virtualenv + python3 -m pip install --upgrade pip python3 ./mach bootstrap brew install gnu-tar - name: Build (${{ inputs.profile }}) diff --git a/.github/workflows/nightly-rust.yml b/.github/workflows/nightly-rust.yml index df31f969b51..10b04f3e9ed 100644 --- a/.github/workflows/nightly-rust.yml +++ b/.github/workflows/nightly-rust.yml @@ -24,7 +24,7 @@ jobs: run: echo nightly > rust-toolchain - name: Bootstrap run: | - python3 -m pip install --upgrade pip virtualenv + python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap - name: Release build diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml index 9b2bea295f0..bc4166b05b0 100644 --- a/.github/workflows/scheduled-wpt-import.yml +++ b/.github/workflows/scheduled-wpt-import.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/download-artifact@v3 - name: Prep environment run: | - python3 -m pip install --upgrade pip virtualenv + python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap - name: Add upstream remote diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b8c18701c61..c2f76078f91 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -78,7 +78,7 @@ jobs: # this point crown is not installed yet. RUSTC: "rustc" run: | - python -m pip install --upgrade pip virtualenv + python -m pip install --upgrade pip python mach fetch python mach bootstrap-gstreamer cargo install --path support/crown diff --git a/.gitignore b/.gitignore index 8622eb8484c..b9785f0beaf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ /ports/android/local.properties /ports/android/obj /python/_virtualenv* +/python/_venv* /python/tidy/servo_tidy.egg-info /tests/wpt/sync *.pkl diff --git a/README.md b/README.md index 9e6c7563663..025b713c64f 100644 --- a/README.md +++ b/README.md @@ -25,18 +25,17 @@ manually, try the [manual build setup][manual-build]. - Install [Xcode](https://developer.apple.com/xcode/) - Install [Homebrew](https://brew.sh/) - Run `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` -- Run `pip3 install virtualenv` - Run `./mach bootstrap`
*Note: This will install the recommended version of GStreamer globally on your system.* ### Linux - Run `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` -- Install Python and virtualenv - - **Debian-like:** Run `sudo apt install python3-virtualenv python3-pip` - - **Fedora:** Run `sudo dnf install python3 python3-virtualenv python3-pip python3-devel` - - **Arch:** Run `sudo pacman -S --needed python python-virtualenv python-pip` - - **Gentoo:** Run `sudo emerge dev-python/virtualenv dev-python/pip` +- Install Python + - **Debian-like:** Run `sudo apt install python3-pip` + - **Fedora:** Run `sudo dnf install python3 python3-pip python3-devel` + - **Arch:** Run `sudo pacman -S --needed python python-pip` + - **Gentoo:** Run `sudo emerge dev-python/pip` - Run `./mach bootstrap` ### Windows diff --git a/etc/ci/performance/README.md b/etc/ci/performance/README.md index 92c20ee5ec5..a269a710858 100644 --- a/etc/ci/performance/README.md +++ b/etc/ci/performance/README.md @@ -113,13 +113,3 @@ If you want to test the data submission code in `submit_to_perfherder.py` withou * `vagrant ssh` * `./manage.py create_credentials "description"`, the email has to match your logged in user. Remember to log-in through the Web UI once before you run this. * Setup your Treeherder client ID and secret as environment variables `TREEHERDER_CLIENT_ID` and `TREEHERDER_CLIENT_SECRET` - -# Troubleshooting - - If you saw this error message: - -``` -venv/bin/activate: line 8: _OLD_VIRTUAL_PATH: unbound variable -``` - -That means your `virtualenv` is too old, try run `pip install -U virtualenv` to upgrade (If you installed ubuntu's `python-virtualenv` package, uninstall it first then install it through `pip`) diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index c8085315219..53f7adb3fc4 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -4,8 +4,9 @@ import os import platform -import sys +import site import shutil +import sys from subprocess import Popen from tempfile import TemporaryFile @@ -80,27 +81,26 @@ CATEGORIES = { } } -# Possible names of executables -# NOTE: Windows Python doesn't provide versioned executables, so we must use -# the plain names. On MSYS, we still use Windows Python. -PYTHON_NAMES = ["python-2.7", "python2.7", "python2", "python"] - - -def _get_exec_path(names, is_valid_path=lambda _path: True): - for name in names: - path = shutil.which(name) - if path and is_valid_path(path): - return path - return None - +# venv calls its scripts folder "bin" on non-Windows and "Scripts" on Windows. def _get_virtualenv_script_dir(): - # Virtualenv calls its scripts folder "bin" on linux/OSX/MSYS64 but "Scripts" on Windows if os.name == "nt" and os.sep != "/": return "Scripts" return "bin" +# venv names its lib folder something like "lib/python3.11/site-packages" on +# non-Windows and "Lib\site-packages" on Window. +def _get_virtualenv_lib_dir(): + if os.name == "nt" and os.sep != "/": + return os.path.join("Lib", "site-packages") + return os.path.join( + "lib", + f"python{sys.version_info[0]}.{sys.version_info[1]}", + "site-packages" + ) + + def _process_exec(args): with TemporaryFile() as out: with TemporaryFile() as err: @@ -130,32 +130,29 @@ def _process_exec(args): def _activate_virtualenv(topdir): - virtualenv_path = os.path.join(topdir, "python", "_virtualenv%d.%d" % (sys.version_info[0], sys.version_info[1])) - python = sys.executable # If there was no python, mach wouldn't have run at all! - if not python: - sys.exit('Failed to find python executable for starting virtualenv.') + virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1])) + python = sys.executable - script_dir = _get_virtualenv_script_dir() - activate_path = os.path.join(virtualenv_path, script_dir, "activate_this.py") - need_pip_upgrade = False - if not (os.path.exists(virtualenv_path) and os.path.exists(activate_path)): - import importlib - try: - importlib.import_module('virtualenv') - except ModuleNotFoundError: - sys.exit("Python virtualenv is not installed. Please install it prior to running mach.") + if os.environ.get("VIRTUAL_ENV") != virtualenv_path: + venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir()) + if not os.path.exists(virtualenv_path): + _process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path]) - _process_exec([python, "-m", "virtualenv", "-p", python, "--system-site-packages", virtualenv_path]) + # This general approach is taken from virtualenv's `activate_this.py`. + os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)]) + os.environ["VIRTUAL_ENV"] = virtualenv_path - # We want to upgrade pip when virtualenv created for the first time - need_pip_upgrade = True + prev_length = len(sys.path) + lib_path = os.path.realpath(os.path.join(virtualenv_path, _get_virtualenv_lib_dir())) + site.addsitedir(lib_path) + sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length] - exec(compile(open(activate_path).read(), activate_path, 'exec'), dict(__file__=activate_path)) + sys.real_prefix = sys.prefix + sys.prefix = virtualenv_path - python = _get_exec_path(PYTHON_NAMES, - is_valid_path=lambda path: path.startswith(virtualenv_path)) - if not python: - sys.exit("Python executable in virtualenv failed to activate.") + # Use the python in our venv for subprocesses, not the python we were originally run with. + # Otherwise pip may still try to write to the wrong site-packages directory. + python = os.path.join(venv_script_path, "python") # TODO: Right now, we iteratively install all the requirements by invoking # `pip install` each time. If it were the case that there were conflicting @@ -168,11 +165,6 @@ def _activate_virtualenv(topdir): os.path.join(WPT_RUNNER_PATH, "requirements.txt",), ] - if need_pip_upgrade: - # Upgrade pip when virtualenv is created to fix the issue - # https://github.com/servo/servo/issues/11074 - _process_exec([python, "-m", "pip", "install", "-I", "-U", "pip"]) - for req_rel_path in requirements_paths: req_path = os.path.join(topdir, req_rel_path) marker_file = req_rel_path.replace(os.path.sep, '-') diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index cd977dfe91d..fae27e25b9e 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -252,7 +252,7 @@ class MachCommands(CommandBase): subprocess.call(["perl", "-i", "-pe", expr, target_path]) @Command('clean', - description='Clean the target/ and python/_virtualenv[version]/ directories', + description='Clean the target/ and python/_venv[version]/ directories', category='build') @CommandArgument('--manifest-path', default=None, @@ -265,7 +265,7 @@ class MachCommands(CommandBase): def clean(self, manifest_path=None, params=[], verbose=False): self.ensure_bootstrapped() - virtualenv_fname = '_virtualenv%d.%d' % (sys.version_info[0], sys.version_info[1]) + virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1]) virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname) if path.exists(virtualenv_path): print('Removing virtualenv directory: %s' % virtualenv_path) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index daf9b68dd8e..af44102b313 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -36,7 +36,6 @@ from xml.etree.ElementTree import XML import toml -from mach_bootstrap import _get_exec_path from mach.decorators import CommandArgument, CommandArgumentGroup from mach.registrar import Registrar @@ -618,8 +617,8 @@ class CommandBase(object): host_suffix = "x86_64" host = os_type + "-" + host_suffix - host_cc = env.get('HOST_CC') or _get_exec_path(["clang"]) or _get_exec_path(["gcc"]) - host_cxx = env.get('HOST_CXX') or _get_exec_path(["clang++"]) or _get_exec_path(["g++"]) + host_cc = env.get('HOST_CC') or shutil.which(["clang"]) or util.whichget_exec_path(["gcc"]) + host_cxx = env.get('HOST_CXX') or util.whichget_exec_path(["clang++"]) or util.whichget_exec_path(["g++"]) llvm_toolchain = path.join(env['ANDROID_NDK'], "toolchains", "llvm", "prebuilt", host) gcc_toolchain = path.join(env['ANDROID_NDK'], "toolchains", diff --git a/servo-tidy.toml b/servo-tidy.toml index 066d2ec6f7e..e1f933bb610 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -105,7 +105,7 @@ directories = [ "./tests/wpt/mozilla/tests/mozilla/referrer-policy", "./tests/wpt/mozilla/tests/webgl", "./python/tidy/tests", - "./python/_virtualenv*", + "./python/_v*", "./python/mach", # Generated and upstream code combined with our own. Could use cleanup "./target",