mach: recreate .venv when .python-version changes (#34712)

The new images published in servo/ci-runners#12 should have `uv`
installed already and the initial build of servo triggered during the
base image construction will force the installation of the Python
version mentioned at the time of the image construction (3.12). When
.python-version changes, we can no longer use the .venv baked into the
image and must recreate the it to avoid activating the environment.

Signed-off-by: Mukilan Thiyagarajan <mukilan@igalia.com>
This commit is contained in:
Mukilan Thiyagarajan 2024-12-20 15:25:21 +05:30 committed by GitHub
parent 50c9c72778
commit adfee3daa5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 46 additions and 46 deletions

View file

@ -57,7 +57,7 @@ jobs:
- name: Setup Python - name: Setup Python
uses: ./.github/actions/setup-python uses: ./.github/actions/setup-python
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: sudo apt update && python3 ./mach bootstrap --skip-lints run: sudo apt update && ./mach bootstrap --skip-lints
- name: Set up JDK 17 - name: Set up JDK 17
uses: actions/setup-java@v4 uses: actions/setup-java@v4
with: with:
@ -95,7 +95,7 @@ jobs:
APK_SIGNING_KEY_ALIAS: ${{ secrets.APK_SIGNING_KEY_ALIAS }} APK_SIGNING_KEY_ALIAS: ${{ secrets.APK_SIGNING_KEY_ALIAS }}
APK_SIGNING_KEY_PASS: ${{ secrets.APK_SIGNING_KEY_PASS }} APK_SIGNING_KEY_PASS: ${{ secrets.APK_SIGNING_KEY_PASS }}
run: | run: |
python3 ./mach build --use-crown --locked --target ${{ matrix.target }} --${{ inputs.profile }} ./mach build --use-crown --locked --target ${{ matrix.target }} --${{ inputs.profile }}
cp -r target/cargo-timings target/cargo-timings-android-${{ matrix.target }} cp -r target/cargo-timings target/cargo-timings-android-${{ matrix.target }}
# TODO: This is disabled since APK crashes during startup. # TODO: This is disabled since APK crashes during startup.
# See https://github.com/servo/servo/issues/31134 # See https://github.com/servo/servo/issues/31134
@ -110,7 +110,7 @@ jobs:
- name: Upload nightly - name: Upload nightly
if: ${{ inputs.upload && contains(matrix.target, 'aarch64') }} if: ${{ inputs.upload && contains(matrix.target, 'aarch64') }}
run: | run: |
python3 ./mach upload-nightly android \ ./mach upload-nightly android \
--secret-from-environment \ --secret-from-environment \
--github-release-id ${{ inputs.github-release-id }} --github-release-id ${{ inputs.github-release-id }}
env: env:

View file

@ -18,11 +18,11 @@ jobs:
- name: Bootstrap - name: Bootstrap
run: | run: |
sudo apt update sudo apt update
python3 ./mach bootstrap --skip-lints ./mach bootstrap --skip-lints
- name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle. - name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle.
run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
- name: Compile docs - name: Compile docs
run: python3 ./mach doc run: ./mach doc
env: env:
RUSTDOCFLAGS: --document-private-items RUSTDOCFLAGS: --document-private-items
- name: Upload docs - name: Upload docs

View file

@ -41,10 +41,10 @@ jobs:
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: | run: |
sudo apt update sudo apt update
python3 ./mach bootstrap ./mach bootstrap
# TODO: Do GitHub anotaions # TODO: Do GitHub anotaions
- name: Clippy - name: Clippy
run: | run: |
python3 ./mach clippy --use-crown --locked -- -- --deny warnings ./mach clippy --use-crown --locked -- -- --deny warnings
- name: Tidy - name: Tidy
run: python3 ./mach test-tidy --no-progress --all run: ./mach test-tidy --no-progress --all

View file

@ -58,7 +58,7 @@ jobs:
run: | run: |
sudo apt update sudo apt update
sudo apt install -qy --no-install-recommends mesa-vulkan-drivers sudo apt install -qy --no-install-recommends mesa-vulkan-drivers
python3 ./mach bootstrap --skip-lints ./mach bootstrap --skip-lints
- name: Sync from upstream WPT - name: Sync from upstream WPT
if: ${{ inputs.wpt-sync-from-upstream }} if: ${{ inputs.wpt-sync-from-upstream }}
run: | run: |
@ -67,7 +67,7 @@ jobs:
run: | run: |
mkdir -p wpt-filtered-logs/linux-${{ inputs.wpt-layout }} mkdir -p wpt-filtered-logs/linux-${{ inputs.wpt-layout }}
mkdir -p wpt-full-logs/linux-${{ inputs.wpt-layout }} mkdir -p wpt-full-logs/linux-${{ inputs.wpt-layout }}
python3 ./mach test-wpt \ ./mach test-wpt \
$WPT_LAYOUT_ARG \ $WPT_LAYOUT_ARG \
$WPT_ALWAYS_SUCCEED_ARG \ $WPT_ALWAYS_SUCCEED_ARG \
--${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \ --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \

View file

@ -129,14 +129,13 @@ jobs:
if: ${{ runner.environment != 'self-hosted' && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly if: ${{ runner.environment != 'self-hosted' && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly
run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
- name: Setup Python - name: Setup Python
if: ${{ runner.environment != 'self-hosted' }}
uses: ./.github/actions/setup-python uses: ./.github/actions/setup-python
with:
skip-python-setup: ${{ runner.environment == 'self-hosted' }}
- name: Bootstrap dependencies - name: Bootstrap dependencies
if: ${{ runner.environment != 'self-hosted' }} if: ${{ runner.environment != 'self-hosted' }}
run: | run: |
sudo apt update sudo apt update
python3 ./mach bootstrap --skip-lints ./mach bootstrap --skip-lints
# Always install crown, even on self-hosted runners, because it is tightly # Always install crown, even on self-hosted runners, because it is tightly
# coupled to the rustc version, and we may have the wrong version if the # coupled to the rustc version, and we may have the wrong version if the
@ -146,10 +145,10 @@ jobs:
- name: Build (${{ inputs.profile }}) - name: Build (${{ inputs.profile }})
run: | run: |
python3 ./mach build --use-crown --locked --${{ inputs.profile }} --features "layout_2013" ./mach build --use-crown --locked --${{ inputs.profile }} --features "layout_2013"
cp -r target/cargo-timings target/cargo-timings-linux cp -r target/cargo-timings target/cargo-timings-linux
- name: Smoketest - name: Smoketest
run: xvfb-run python3 ./mach smoketest --${{ inputs.profile }} run: xvfb-run ./mach smoketest --${{ inputs.profile }}
- name: Script tests - name: Script tests
run: ./mach test-scripts run: ./mach test-scripts
- name: Unit tests - name: Unit tests
@ -158,7 +157,7 @@ jobs:
with: with:
timeout_minutes: 20 timeout_minutes: 20
max_attempts: 2 # https://github.com/servo/servo/issues/30683 max_attempts: 2 # https://github.com/servo/servo/issues/30683
command: python ./mach test-unit --${{ inputs.profile }} command: ./mach test-unit --${{ inputs.profile }}
- name: Archive build timing - name: Archive build timing
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
@ -166,7 +165,7 @@ jobs:
# Using a wildcard here ensures that the archive includes the path. # Using a wildcard here ensures that the archive includes the path.
path: target/cargo-timings-* path: target/cargo-timings-*
- name: Build mach package - name: Build mach package
run: python3 ./mach package --${{ inputs.profile }} run: ./mach package --${{ inputs.profile }}
- name: Upload artifact for mach package - name: Upload artifact for mach package
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
@ -175,7 +174,7 @@ jobs:
- name: Upload nightly - name: Upload nightly
if: ${{ inputs.upload }} if: ${{ inputs.upload }}
run: | run: |
python3 ./mach upload-nightly linux \ ./mach upload-nightly linux \
--secret-from-environment \ --secret-from-environment \
--github-release-id ${{ inputs.github-release-id }} --github-release-id ${{ inputs.github-release-id }}
env: env:

View file

@ -46,14 +46,14 @@ jobs:
- name: Prep test environment - name: Prep test environment
run: | run: |
gtar -xzf target.tar.gz gtar -xzf target.tar.gz
python3 ./mach bootstrap --skip-lints ./mach bootstrap --skip-lints
- name: Smoketest - name: Smoketest
run: python3 ./mach smoketest --${{ inputs.profile }} run: ./mach smoketest --${{ inputs.profile }}
- name: Run tests - name: Run tests
run: | run: |
mkdir -p wpt-filtered-logs/macos-${{ inputs.wpt-layout }} mkdir -p wpt-filtered-logs/macos-${{ inputs.wpt-layout }}
mkdir -p wpt-full-logs/macos-${{ inputs.wpt-layout }} mkdir -p wpt-full-logs/macos-${{ inputs.wpt-layout }}
python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ ./mach test-wpt $WPT_COMMAND_LINE_ARG \
--${{ inputs.profile }} --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \ --${{ inputs.profile }} --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \
--total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
--log-raw wpt-full-logs/macos-${{ inputs.wpt-layout }}/${{ matrix.chunk_id }}.log \ --log-raw wpt-full-logs/macos-${{ inputs.wpt-layout }}/${{ matrix.chunk_id }}.log \

View file

@ -80,18 +80,18 @@ jobs:
run: cargo install --path support/crown run: cargo install --path support/crown
- name: Bootstrap - name: Bootstrap
run: | run: |
python3 ./mach bootstrap --skip-lints ./mach bootstrap --skip-lints
brew install gnu-tar brew install gnu-tar
- name: Build (${{ inputs.profile }}) - name: Build (${{ inputs.profile }})
run: | run: |
python3 ./mach build --use-crown --locked --${{ inputs.profile }} ./mach build --use-crown --locked --${{ inputs.profile }}
cp -r target/cargo-timings target/cargo-timings-macos cp -r target/cargo-timings target/cargo-timings-macos
- name: Smoketest - name: Smoketest
uses: nick-fields/retry@v3 uses: nick-fields/retry@v3
with: # See https://github.com/servo/servo/issues/30757 with: # See https://github.com/servo/servo/issues/30757
timeout_minutes: 5 timeout_minutes: 5
max_attempts: 2 max_attempts: 2
command: python3 ./mach smoketest --${{ inputs.profile }} command: ./mach smoketest --${{ inputs.profile }}
- name: Script tests - name: Script tests
run: ./mach test-scripts run: ./mach test-scripts
- name: Unit tests - name: Unit tests
@ -100,9 +100,9 @@ jobs:
with: with:
timeout_minutes: 40 # https://github.com/servo/servo/issues/30275 timeout_minutes: 40 # https://github.com/servo/servo/issues/30275
max_attempts: 3 # https://github.com/servo/servo/issues/30683 max_attempts: 3 # https://github.com/servo/servo/issues/30683
command: python3 ./mach test-unit --${{ inputs.profile }} command: ./mach test-unit --${{ inputs.profile }}
- name: Build mach package - name: Build mach package
run: python3 ./mach package --${{ inputs.profile }} run: ./mach package --${{ inputs.profile }}
- name: Run DMG smoketest - name: Run DMG smoketest
uses: nick-fields/retry@v3 uses: nick-fields/retry@v3
with: # See https://github.com/servo/servo/issues/30757 with: # See https://github.com/servo/servo/issues/30757
@ -123,7 +123,7 @@ jobs:
- name: Upload nightly - name: Upload nightly
if: ${{ inputs.upload }} if: ${{ inputs.upload }}
run: | run: |
python3 ./mach upload-nightly mac --secret-from-environment \ ./mach upload-nightly mac --secret-from-environment \
--github-release-id ${{ inputs.github-release-id }} --github-release-id ${{ inputs.github-release-id }}
env: env:
S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}

View file

@ -51,7 +51,7 @@ jobs:
- name: Setup Python - name: Setup Python
uses: ./.github/actions/setup-python uses: ./.github/actions/setup-python
- name: Bootstrap dependencies - name: Bootstrap dependencies
run: sudo apt update && python3 ./mach bootstrap --skip-lints run: sudo apt update && ./mach bootstrap --skip-lints
- name: Setup OpenHarmony SDK - name: Setup OpenHarmony SDK
id: setup_sdk id: setup_sdk
uses: openharmony-rs/setup-ohos-sdk@v0.1 uses: openharmony-rs/setup-ohos-sdk@v0.1
@ -83,7 +83,7 @@ jobs:
OHOS_SDK_NATIVE: ${{ steps.setup_sdk.outputs.ohos_sdk_native }} OHOS_SDK_NATIVE: ${{ steps.setup_sdk.outputs.ohos_sdk_native }}
OHOS_BASE_SDK_HOME: ${{ steps.setup_sdk.outputs.ohos-base-sdk-home }} OHOS_BASE_SDK_HOME: ${{ steps.setup_sdk.outputs.ohos-base-sdk-home }}
run: | run: |
python3 ./mach build --locked --target ${{ matrix.target }} --${{ inputs.profile }} ./mach build --locked --target ${{ matrix.target }} --${{ inputs.profile }}
cp -r target/cargo-timings target/cargo-timings-ohos-${{ matrix.target }} cp -r target/cargo-timings target/cargo-timings-ohos-${{ matrix.target }}
- name: Archive build timing - name: Archive build timing
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
@ -94,7 +94,7 @@ jobs:
- name: Upload nightly - name: Upload nightly
if: ${{ inputs.upload && contains(matrix.target, 'aarch64') }} if: ${{ inputs.upload && contains(matrix.target, 'aarch64') }}
run: | run: |
python3 ./mach upload-nightly ohos \ ./mach upload-nightly ohos \
--secret-from-environment \ --secret-from-environment \
--github-release-id ${{ inputs.github-release-id }} --github-release-id ${{ inputs.github-release-id }}
env: env:

View file

@ -40,7 +40,7 @@ jobs:
- name: Prep environment - name: Prep environment
run: | run: |
sudo apt update sudo apt update
python3 ./mach bootstrap ./mach bootstrap
- name: Add upstream remote - name: Add upstream remote
run: | run: |
git config --local user.email "ghbot+wpt-sync@servo.org" git config --local user.email "ghbot+wpt-sync@servo.org"

View file

@ -115,16 +115,14 @@ jobs:
run: | run: |
choco install wixtoolset choco install wixtoolset
echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- uses: actions/setup-python@v5
- name: Setup Python - name: Setup Python
if: ${{ runner.environment != 'self-hosted' }}
uses: ./.github/actions/setup-python uses: ./.github/actions/setup-python
with:
skip-python-setup: ${{ runner.environment == 'self-hosted' }}
- name: Bootstrap - name: Bootstrap
if: ${{ runner.environment != 'self-hosted' }} if: ${{ runner.environment != 'self-hosted' }}
run: | run: |
python mach fetch .\mach fetch
python mach bootstrap-gstreamer .\mach bootstrap-gstreamer
# For some reason WiX isn't currently on the GitHub runner path. This is a # For some reason WiX isn't currently on the GitHub runner path. This is a
# temporary workaround until that is fixed. # temporary workaround until that is fixed.
- name: Add WiX to Path - name: Add WiX to Path
@ -147,21 +145,21 @@ jobs:
- name: Build (${{ inputs.profile }}) - name: Build (${{ inputs.profile }})
run: | run: |
python mach build --use-crown --locked --${{ inputs.profile }} .\mach build --use-crown --locked --${{ inputs.profile }}
cp C:\a\servo\servo\target\cargo-timings C:\a\servo\servo\target\cargo-timings-windows -Recurse cp C:\a\servo\servo\target\cargo-timings C:\a\servo\servo\target\cargo-timings-windows -Recurse
- name: Copy resources - name: Copy resources
if: ${{ runner.environment != 'self-hosted' }} if: ${{ runner.environment != 'self-hosted' }}
# GitHub-hosted runners check out the repo on D: drive. # GitHub-hosted runners check out the repo on D: drive.
run: cp D:\a\servo\servo\resources C:\a\servo\servo -Recurse run: cp D:\a\servo\servo\resources C:\a\servo\servo -Recurse
- name: Smoketest - name: Smoketest
run: python mach smoketest --${{ inputs.profile }} run: .\mach smoketest --${{ inputs.profile }}
- name: Unit tests - name: Unit tests
if: ${{ inputs.unit-tests }} if: ${{ inputs.unit-tests }}
uses: nick-fields/retry@v3 uses: nick-fields/retry@v3
with: with:
timeout_minutes: 30 timeout_minutes: 30
max_attempts: 3 # https://github.com/servo/servo/issues/30683 max_attempts: 3 # https://github.com/servo/servo/issues/30683
command: python mach test-unit --${{ inputs.profile }} -- -- --test-threads=1 command: .\mach test-unit --${{ inputs.profile }} -- -- --test-threads=1
- name: Archive build timing - name: Archive build timing
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
@ -169,7 +167,7 @@ jobs:
# Using a wildcard here ensures that the archive includes the path. # Using a wildcard here ensures that the archive includes the path.
path: C:\\a\\servo\\servo\\target\\cargo-timings-* path: C:\\a\\servo\\servo\\target\\cargo-timings-*
- name: Build mach package - name: Build mach package
run: python mach package --${{ inputs.profile }} run: .\mach package --${{ inputs.profile }}
- name: Upload artifact for mach package - name: Upload artifact for mach package
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
@ -182,7 +180,7 @@ jobs:
- name: Upload nightly - name: Upload nightly
if: ${{ inputs.upload }} if: ${{ inputs.upload }}
run: | run: |
python mach upload-nightly windows-msvc --secret-from-environment ` .\mach upload-nightly windows-msvc --secret-from-environment `
--github-release-id ${{ inputs.github-release-id }} --github-release-id ${{ inputs.github-release-id }}
env: env:
S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}

View file

@ -89,7 +89,7 @@ def _process_exec(args, cwd):
sys.exit(1) sys.exit(1)
def install_virtual_env_requirements(project_path: str, virtualenv_path: str): def install_virtual_env_requirements(project_path: str, marker_path: str):
requirements_paths = [ requirements_paths = [
os.path.join(project_path, "python", "requirements.txt"), os.path.join(project_path, "python", "requirements.txt"),
os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",), os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",),
@ -102,7 +102,6 @@ def install_virtual_env_requirements(project_path: str, virtualenv_path: str):
requirements_hasher.update(file.read()) requirements_hasher.update(file.read())
try: try:
marker_path = os.path.join(virtualenv_path, "requirements.sha256")
with open(marker_path, 'r') as marker_file: with open(marker_path, 'r') as marker_file:
marker_hash = marker_file.read() marker_hash = marker_file.read()
except FileNotFoundError: except FileNotFoundError:
@ -123,15 +122,19 @@ def install_virtual_env_requirements(project_path: str, virtualenv_path: str):
def _activate_virtualenv(topdir): def _activate_virtualenv(topdir):
virtualenv_path = os.path.join(topdir, ".venv") virtualenv_path = os.path.join(topdir, ".venv")
with open(".python-version", "r") as python_version_file:
required_python_version = python_version_file.read().strip()
marker_path = os.path.join(virtualenv_path, f"requirements.{required_python_version}.sha256")
if os.environ.get("VIRTUAL_ENV") != virtualenv_path: if os.environ.get("VIRTUAL_ENV") != virtualenv_path:
if not os.path.exists(virtualenv_path): if not os.path.exists(marker_path):
print(" * Setting up virtual environment...") print(" * Setting up virtual environment...")
_process_exec(["uv", "venv"], cwd=topdir) _process_exec(["uv", "venv"], cwd=topdir)
script_dir = "Scripts" if _is_windows() else "bin" script_dir = "Scripts" if _is_windows() else "bin"
runpy.run_path(os.path.join(virtualenv_path, script_dir, 'activate_this.py')) runpy.run_path(os.path.join(virtualenv_path, script_dir, 'activate_this.py'))
install_virtual_env_requirements(topdir, virtualenv_path) install_virtual_env_requirements(topdir, marker_path)
# Turn off warnings about deprecated syntax in our indirect dependencies. # Turn off warnings about deprecated syntax in our indirect dependencies.
# TODO: Find a better approach for doing this. # TODO: Find a better approach for doing this.

View file

@ -89,7 +89,7 @@ stdenv.mkDerivation (androidEnvironment // {
# that `uv` will just symlink to the one in nix store. Otherwise `uv` will # that `uv` will just symlink to the one in nix store. Otherwise `uv` will
# download a pre-built binary that won't work on nix. # download a pre-built binary that won't work on nix.
# FIXME: dbus python module needs to be installed into the virtual environment. # FIXME: dbus python module needs to be installed into the virtual environment.
python312 python311
uv uv
# This pins gnumake to 4.3 since 4.4 breaks jobserver # This pins gnumake to 4.3 since 4.4 breaks jobserver