From adfee3daa536f75bff7074d2f8e5e1a4863d983b Mon Sep 17 00:00:00 2001 From: Mukilan Thiyagarajan Date: Fri, 20 Dec 2024 15:25:21 +0530 Subject: [PATCH] mach: recreate .venv when .python-version changes (#34712) The new images published in servo/ci-runners#12 should have `uv` installed already and the initial build of servo triggered during the base image construction will force the installation of the Python version mentioned at the time of the image construction (3.12). When .python-version changes, we can no longer use the .venv baked into the image and must recreate the it to avoid activating the environment. Signed-off-by: Mukilan Thiyagarajan --- .github/workflows/android.yml | 6 +++--- .github/workflows/docs.yml | 4 ++-- .github/workflows/lint.yml | 6 +++--- .github/workflows/linux-wpt.yml | 4 ++-- .github/workflows/linux.yml | 15 +++++++-------- .github/workflows/mac-wpt.yml | 6 +++--- .github/workflows/mac.yml | 12 ++++++------ .github/workflows/ohos.yml | 6 +++--- .github/workflows/scheduled-wpt-import.yml | 2 +- .github/workflows/windows.yml | 18 ++++++++---------- python/mach_bootstrap.py | 11 +++++++---- shell.nix | 2 +- 12 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 678fe98beb8..ad0adda94cc 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -57,7 +57,7 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-python - name: Bootstrap dependencies - run: sudo apt update && python3 ./mach bootstrap --skip-lints + run: sudo apt update && ./mach bootstrap --skip-lints - name: Set up JDK 17 uses: actions/setup-java@v4 with: @@ -95,7 +95,7 @@ jobs: APK_SIGNING_KEY_ALIAS: ${{ secrets.APK_SIGNING_KEY_ALIAS }} APK_SIGNING_KEY_PASS: ${{ secrets.APK_SIGNING_KEY_PASS }} run: | - python3 ./mach build --use-crown --locked --target ${{ matrix.target }} --${{ inputs.profile }} + ./mach build --use-crown --locked --target ${{ matrix.target }} --${{ inputs.profile }} cp -r target/cargo-timings target/cargo-timings-android-${{ matrix.target }} # TODO: This is disabled since APK crashes during startup. # See https://github.com/servo/servo/issues/31134 @@ -110,7 +110,7 @@ jobs: - name: Upload nightly if: ${{ inputs.upload && contains(matrix.target, 'aarch64') }} run: | - python3 ./mach upload-nightly android \ + ./mach upload-nightly android \ --secret-from-environment \ --github-release-id ${{ inputs.github-release-id }} env: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 74530c40e1f..448486ccd49 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,11 +18,11 @@ jobs: - name: Bootstrap run: | sudo apt update - python3 ./mach bootstrap --skip-lints + ./mach bootstrap --skip-lints - name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle. run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - name: Compile docs - run: python3 ./mach doc + run: ./mach doc env: RUSTDOCFLAGS: --document-private-items - name: Upload docs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index aea6c72e638..a7463d6425c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,10 +41,10 @@ jobs: - name: Bootstrap dependencies run: | sudo apt update - python3 ./mach bootstrap + ./mach bootstrap # TODO: Do GitHub anotaions - name: Clippy run: | - python3 ./mach clippy --use-crown --locked -- -- --deny warnings + ./mach clippy --use-crown --locked -- -- --deny warnings - name: Tidy - run: python3 ./mach test-tidy --no-progress --all + run: ./mach test-tidy --no-progress --all diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index f710518b462..5fed32d1009 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -58,7 +58,7 @@ jobs: run: | sudo apt update sudo apt install -qy --no-install-recommends mesa-vulkan-drivers - python3 ./mach bootstrap --skip-lints + ./mach bootstrap --skip-lints - name: Sync from upstream WPT if: ${{ inputs.wpt-sync-from-upstream }} run: | @@ -67,7 +67,7 @@ jobs: run: | mkdir -p wpt-filtered-logs/linux-${{ inputs.wpt-layout }} mkdir -p wpt-full-logs/linux-${{ inputs.wpt-layout }} - python3 ./mach test-wpt \ + ./mach test-wpt \ $WPT_LAYOUT_ARG \ $WPT_ALWAYS_SUCCEED_ARG \ --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \ diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 192623c6223..9f1ebbf1e1b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -129,14 +129,13 @@ jobs: if: ${{ runner.environment != 'self-hosted' && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - name: Setup Python + if: ${{ runner.environment != 'self-hosted' }} uses: ./.github/actions/setup-python - with: - skip-python-setup: ${{ runner.environment == 'self-hosted' }} - name: Bootstrap dependencies if: ${{ runner.environment != 'self-hosted' }} run: | sudo apt update - python3 ./mach bootstrap --skip-lints + ./mach bootstrap --skip-lints # Always install crown, even on self-hosted runners, because it is tightly # coupled to the rustc version, and we may have the wrong version if the @@ -146,10 +145,10 @@ jobs: - name: Build (${{ inputs.profile }}) run: | - python3 ./mach build --use-crown --locked --${{ inputs.profile }} --features "layout_2013" + ./mach build --use-crown --locked --${{ inputs.profile }} --features "layout_2013" cp -r target/cargo-timings target/cargo-timings-linux - name: Smoketest - run: xvfb-run python3 ./mach smoketest --${{ inputs.profile }} + run: xvfb-run ./mach smoketest --${{ inputs.profile }} - name: Script tests run: ./mach test-scripts - name: Unit tests @@ -158,7 +157,7 @@ jobs: with: timeout_minutes: 20 max_attempts: 2 # https://github.com/servo/servo/issues/30683 - command: python ./mach test-unit --${{ inputs.profile }} + command: ./mach test-unit --${{ inputs.profile }} - name: Archive build timing uses: actions/upload-artifact@v4 with: @@ -166,7 +165,7 @@ jobs: # Using a wildcard here ensures that the archive includes the path. path: target/cargo-timings-* - name: Build mach package - run: python3 ./mach package --${{ inputs.profile }} + run: ./mach package --${{ inputs.profile }} - name: Upload artifact for mach package uses: actions/upload-artifact@v4 with: @@ -175,7 +174,7 @@ jobs: - name: Upload nightly if: ${{ inputs.upload }} run: | - python3 ./mach upload-nightly linux \ + ./mach upload-nightly linux \ --secret-from-environment \ --github-release-id ${{ inputs.github-release-id }} env: diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml index 610da047a5f..6c5e527cfd4 100644 --- a/.github/workflows/mac-wpt.yml +++ b/.github/workflows/mac-wpt.yml @@ -46,14 +46,14 @@ jobs: - name: Prep test environment run: | gtar -xzf target.tar.gz - python3 ./mach bootstrap --skip-lints + ./mach bootstrap --skip-lints - name: Smoketest - run: python3 ./mach smoketest --${{ inputs.profile }} + run: ./mach smoketest --${{ inputs.profile }} - name: Run tests run: | mkdir -p wpt-filtered-logs/macos-${{ inputs.wpt-layout }} mkdir -p wpt-full-logs/macos-${{ inputs.wpt-layout }} - python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ + ./mach test-wpt $WPT_COMMAND_LINE_ARG \ --${{ inputs.profile }} --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw wpt-full-logs/macos-${{ inputs.wpt-layout }}/${{ matrix.chunk_id }}.log \ diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 5648c8539e1..a91b29b4b64 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -80,18 +80,18 @@ jobs: run: cargo install --path support/crown - name: Bootstrap run: | - python3 ./mach bootstrap --skip-lints + ./mach bootstrap --skip-lints brew install gnu-tar - name: Build (${{ inputs.profile }}) run: | - python3 ./mach build --use-crown --locked --${{ inputs.profile }} + ./mach build --use-crown --locked --${{ inputs.profile }} cp -r target/cargo-timings target/cargo-timings-macos - name: Smoketest uses: nick-fields/retry@v3 with: # See https://github.com/servo/servo/issues/30757 timeout_minutes: 5 max_attempts: 2 - command: python3 ./mach smoketest --${{ inputs.profile }} + command: ./mach smoketest --${{ inputs.profile }} - name: Script tests run: ./mach test-scripts - name: Unit tests @@ -100,9 +100,9 @@ jobs: with: timeout_minutes: 40 # https://github.com/servo/servo/issues/30275 max_attempts: 3 # https://github.com/servo/servo/issues/30683 - command: python3 ./mach test-unit --${{ inputs.profile }} + command: ./mach test-unit --${{ inputs.profile }} - name: Build mach package - run: python3 ./mach package --${{ inputs.profile }} + run: ./mach package --${{ inputs.profile }} - name: Run DMG smoketest uses: nick-fields/retry@v3 with: # See https://github.com/servo/servo/issues/30757 @@ -123,7 +123,7 @@ jobs: - name: Upload nightly if: ${{ inputs.upload }} run: | - python3 ./mach upload-nightly mac --secret-from-environment \ + ./mach upload-nightly mac --secret-from-environment \ --github-release-id ${{ inputs.github-release-id }} env: S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} diff --git a/.github/workflows/ohos.yml b/.github/workflows/ohos.yml index f599da4013b..78afcfd9216 100644 --- a/.github/workflows/ohos.yml +++ b/.github/workflows/ohos.yml @@ -51,7 +51,7 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-python - name: Bootstrap dependencies - run: sudo apt update && python3 ./mach bootstrap --skip-lints + run: sudo apt update && ./mach bootstrap --skip-lints - name: Setup OpenHarmony SDK id: setup_sdk uses: openharmony-rs/setup-ohos-sdk@v0.1 @@ -83,7 +83,7 @@ jobs: OHOS_SDK_NATIVE: ${{ steps.setup_sdk.outputs.ohos_sdk_native }} OHOS_BASE_SDK_HOME: ${{ steps.setup_sdk.outputs.ohos-base-sdk-home }} run: | - python3 ./mach build --locked --target ${{ matrix.target }} --${{ inputs.profile }} + ./mach build --locked --target ${{ matrix.target }} --${{ inputs.profile }} cp -r target/cargo-timings target/cargo-timings-ohos-${{ matrix.target }} - name: Archive build timing uses: actions/upload-artifact@v4 @@ -94,7 +94,7 @@ jobs: - name: Upload nightly if: ${{ inputs.upload && contains(matrix.target, 'aarch64') }} run: | - python3 ./mach upload-nightly ohos \ + ./mach upload-nightly ohos \ --secret-from-environment \ --github-release-id ${{ inputs.github-release-id }} env: diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml index 145133b817c..02db5683853 100644 --- a/.github/workflows/scheduled-wpt-import.yml +++ b/.github/workflows/scheduled-wpt-import.yml @@ -40,7 +40,7 @@ jobs: - name: Prep environment run: | sudo apt update - python3 ./mach bootstrap + ./mach bootstrap - name: Add upstream remote run: | git config --local user.email "ghbot+wpt-sync@servo.org" diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 8d07de54ae0..4fcbb211695 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -115,16 +115,14 @@ jobs: run: | choco install wixtoolset echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - uses: actions/setup-python@v5 - name: Setup Python + if: ${{ runner.environment != 'self-hosted' }} uses: ./.github/actions/setup-python - with: - skip-python-setup: ${{ runner.environment == 'self-hosted' }} - name: Bootstrap if: ${{ runner.environment != 'self-hosted' }} run: | - python mach fetch - python mach bootstrap-gstreamer + .\mach fetch + .\mach bootstrap-gstreamer # For some reason WiX isn't currently on the GitHub runner path. This is a # temporary workaround until that is fixed. - name: Add WiX to Path @@ -147,21 +145,21 @@ jobs: - name: Build (${{ inputs.profile }}) run: | - python mach build --use-crown --locked --${{ inputs.profile }} + .\mach build --use-crown --locked --${{ inputs.profile }} cp C:\a\servo\servo\target\cargo-timings C:\a\servo\servo\target\cargo-timings-windows -Recurse - name: Copy resources if: ${{ runner.environment != 'self-hosted' }} # GitHub-hosted runners check out the repo on D: drive. run: cp D:\a\servo\servo\resources C:\a\servo\servo -Recurse - name: Smoketest - run: python mach smoketest --${{ inputs.profile }} + run: .\mach smoketest --${{ inputs.profile }} - name: Unit tests if: ${{ inputs.unit-tests }} uses: nick-fields/retry@v3 with: timeout_minutes: 30 max_attempts: 3 # https://github.com/servo/servo/issues/30683 - command: python mach test-unit --${{ inputs.profile }} -- -- --test-threads=1 + command: .\mach test-unit --${{ inputs.profile }} -- -- --test-threads=1 - name: Archive build timing uses: actions/upload-artifact@v4 with: @@ -169,7 +167,7 @@ jobs: # Using a wildcard here ensures that the archive includes the path. path: C:\\a\\servo\\servo\\target\\cargo-timings-* - name: Build mach package - run: python mach package --${{ inputs.profile }} + run: .\mach package --${{ inputs.profile }} - name: Upload artifact for mach package uses: actions/upload-artifact@v4 with: @@ -182,7 +180,7 @@ jobs: - name: Upload nightly if: ${{ inputs.upload }} run: | - python mach upload-nightly windows-msvc --secret-from-environment ` + .\mach upload-nightly windows-msvc --secret-from-environment ` --github-release-id ${{ inputs.github-release-id }} env: S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 14f28f512ce..4e6ace1ffeb 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -89,7 +89,7 @@ def _process_exec(args, cwd): sys.exit(1) -def install_virtual_env_requirements(project_path: str, virtualenv_path: str): +def install_virtual_env_requirements(project_path: str, marker_path: str): requirements_paths = [ os.path.join(project_path, "python", "requirements.txt"), os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",), @@ -102,7 +102,6 @@ def install_virtual_env_requirements(project_path: str, virtualenv_path: str): requirements_hasher.update(file.read()) try: - marker_path = os.path.join(virtualenv_path, "requirements.sha256") with open(marker_path, 'r') as marker_file: marker_hash = marker_file.read() except FileNotFoundError: @@ -123,15 +122,19 @@ def install_virtual_env_requirements(project_path: str, virtualenv_path: str): def _activate_virtualenv(topdir): virtualenv_path = os.path.join(topdir, ".venv") + with open(".python-version", "r") as python_version_file: + required_python_version = python_version_file.read().strip() + marker_path = os.path.join(virtualenv_path, f"requirements.{required_python_version}.sha256") + if os.environ.get("VIRTUAL_ENV") != virtualenv_path: - if not os.path.exists(virtualenv_path): + if not os.path.exists(marker_path): print(" * Setting up virtual environment...") _process_exec(["uv", "venv"], cwd=topdir) script_dir = "Scripts" if _is_windows() else "bin" runpy.run_path(os.path.join(virtualenv_path, script_dir, 'activate_this.py')) - install_virtual_env_requirements(topdir, virtualenv_path) + install_virtual_env_requirements(topdir, marker_path) # Turn off warnings about deprecated syntax in our indirect dependencies. # TODO: Find a better approach for doing this. diff --git a/shell.nix b/shell.nix index 375a22c48f7..fe1efcb6aa7 100644 --- a/shell.nix +++ b/shell.nix @@ -89,7 +89,7 @@ stdenv.mkDerivation (androidEnvironment // { # that `uv` will just symlink to the one in nix store. Otherwise `uv` will # download a pre-built binary that won't work on nix. # FIXME: dbus python module needs to be installed into the virtual environment. - python312 + python311 uv # This pins gnumake to 4.3 since 4.4 breaks jobserver