From d86e713a9cb5be2555d63bd477d47d440fa8c832 Mon Sep 17 00:00:00 2001 From: Martin Robinson Date: Wed, 17 Jan 2024 11:53:34 +0100 Subject: [PATCH] build: Clean up post-build copy of Windows DLLs (#31092) * build: Clean up post-build copy of Windows DLLs - No longer use vcvarsall.bat at all. Instead find the Windows SDK directory by looking in the registry. - Split logic for copying Windows dependencies into its own function and do some minor clean up, such as collecting all MSVC functionality into visual_studio.py. - Remove support for Visual Studio 2015 and Visual Studio 2017. This is a preparatory change in order to support Visual Studio 2022. * More cleanup of the code --- python/servo/build_commands.py | 151 +++++++++------------------------ python/servo/command_base.py | 72 ---------------- python/servo/visual_studio.py | 149 ++++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 183 deletions(-) create mode 100644 python/servo/visual_studio.py diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 9935e2c1c48..f61c82fe5b3 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -8,9 +8,9 @@ # except according to those terms. import datetime -import locale import os import os.path as path +import pathlib import shutil import stat import subprocess @@ -35,6 +35,7 @@ import servo.util from servo.command_base import BuildType, CommandBase, call, check_call from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins +from python.servo.visual_studio import find_msvc_redist_dirs @CommandProvider @@ -80,9 +81,6 @@ class MachCommands(CommandBase): build_start = time() host = servo.platform.host_triple() - if 'windows' in host: - vs_dirs = self.vs_dirs() - target_triple = self.cross_compile_target or servo.platform.host_triple() if host != target_triple and 'windows' in target_triple: if os.environ.get('VisualStudioVersion') or os.environ.get('VCINSTALLDIR'): @@ -91,29 +89,6 @@ class MachCommands(CommandBase): "Visual Studio shell, and make sure the VisualStudioVersion and " "VCINSTALLDIR environment variables are not set.") sys.exit(1) - vcinstalldir = vs_dirs['vcdir'] - if not os.path.exists(vcinstalldir): - print("Can't find Visual C++ %s installation at %s." % (vs_dirs['vs_version'], vcinstalldir)) - sys.exit(1) - - env['PKG_CONFIG_ALLOW_CROSS'] = "1" - - if 'windows' in host: - process = subprocess.Popen('("%s" %s > nul) && "python" -c "import os; print(repr(os.environ))"' % - (os.path.join(vs_dirs['vcdir'], "Auxiliary", "Build", "vcvarsall.bat"), "x64"), - stdout=subprocess.PIPE, shell=True) - stdout, stderr = process.communicate() - exitcode = process.wait() - encoding = locale.getpreferredencoding() # See https://stackoverflow.com/a/9228117 - if exitcode == 0: - decoded = stdout.decode(encoding) - if decoded.startswith("environ("): - decoded = decoded.strip()[8:-1] - os.environ.update(eval(decoded)) - else: - print("Failed to run vcvarsall. stderr:") - print(stderr.decode(encoding)) - exit(1) # Gather Cargo build timings (https://doc.rust-lang.org/cargo/reference/timings.html). opts = ["--timings"] + opts @@ -130,6 +105,7 @@ class MachCommands(CommandBase): ) # Do some additional things if the build succeeded + built_binary = self.get_binary_path(build_type, target=self.cross_compile_target, simpleservo=libsimpleservo) if status == 0: if self.is_android_build and not no_package: flavor = None @@ -143,43 +119,7 @@ class MachCommands(CommandBase): return rv if sys.platform == "win32": - servo_exe_dir = os.path.dirname( - self.get_binary_path(build_type, target=self.cross_compile_target, simpleservo=libsimpleservo) - ) - assert os.path.exists(servo_exe_dir) - - build_path = path.join(servo_exe_dir, "build") - assert os.path.exists(build_path) - - # on msvc, we need to copy in some DLLs in to the servo.exe dir and the directory for unit tests. - def package_generated_shared_libraries(libs, build_path, servo_exe_dir): - for root, dirs, files in os.walk(build_path): - remaining_libs = list(libs) - for lib in libs: - if lib in files: - shutil.copy(path.join(root, lib), servo_exe_dir) - remaining_libs.remove(lib) - continue - libs = remaining_libs - if not libs: - return True - for lib in libs: - print("WARNING: could not find " + lib) - - print("Packaging EGL DLLs") - egl_libs = ["libEGL.dll", "libGLESv2.dll"] - if not package_generated_shared_libraries(egl_libs, build_path, servo_exe_dir): - status = 1 - - # copy needed gstreamer DLLs in to servo.exe dir - if self.enable_media: - print("Packaging gstreamer DLLs") - if not package_gstreamer_dlls(env, servo_exe_dir, target_triple): - status = 1 - - # UWP app packaging already bundles all required DLLs for us. - print("Packaging MSVC DLLs") - if not package_msvc_dlls(servo_exe_dir, target_triple, vs_dirs['vcdir'], vs_dirs['vs_version']): + if not copy_windows_dlls_to_build_directory(built_binary, target_triple): status = 1 elif sys.platform == "darwin": @@ -423,7 +363,38 @@ def package_gstreamer_dylibs(cross_compilation_target, servo_bin): return True -def package_gstreamer_dlls(env, servo_exe_dir, target): +def copy_windows_dlls_to_build_directory(servo_binary: str, target_triple: str) -> bool: + servo_exe_dir = os.path.dirname(servo_binary) + assert os.path.exists(servo_exe_dir) + + build_path = path.join(servo_exe_dir, "build") + assert os.path.exists(build_path) + + # Copy in the built EGL and GLES libraries from where they were built to + # the final build dirctory + def find_and_copy_built_dll(dll_name): + try: + file_to_copy = next(pathlib.Path(build_path).rglob(dll_name)) + shutil.copy(file_to_copy, servo_exe_dir) + except StopIteration: + print(f"WARNING: could not find {dll_name}") + + print(" • Copying ANGLE DLLs to binary directory...") + find_and_copy_built_dll("libEGL.dll") + find_and_copy_built_dll("libGLESv2.dll") + + print(" • Copying GStreamer DLLs to binary directory...") + if not package_gstreamer_dlls(servo_exe_dir, target_triple): + return False + + print(" • Copying MSVC DLLs to binary directory...") + if not package_msvc_dlls(servo_exe_dir, target_triple): + return False + + return True + + +def package_gstreamer_dlls(servo_exe_dir: str, target: str): gst_root = servo.platform.get().gstreamer_root(cross_compilation_target=target) if not gst_root: print("Could not find GStreamer installation directory.") @@ -462,58 +433,16 @@ def package_gstreamer_dlls(env, servo_exe_dir, target): return not missing -def package_msvc_dlls(servo_exe_dir, target, vcinstalldir, vs_version): - # copy some MSVC DLLs to servo.exe dir - msvc_redist_dir = None - vs_platforms = { - "x86_64": "x64", - "i686": "x86", - "aarch64": "arm64", - } - target_arch = target.split('-')[0] - vs_platform = vs_platforms[target_arch] - vc_dir = vcinstalldir or os.environ.get("VCINSTALLDIR", "") - if not vs_version: - vs_version = os.environ.get("VisualStudioVersion", "") +def package_msvc_dlls(servo_exe_dir, target): msvc_deps = [ "msvcp140.dll", "vcruntime140.dll", ] - if target_arch != "aarch64" and vs_version in ("14.0", "15.0", "16.0"): + if "aarch64" not in target != "aarch64": msvc_deps += ["api-ms-win-crt-runtime-l1-1-0.dll"] - # Check if it's Visual C++ Build Tools or Visual Studio 2015 - vs14_vcvars = path.join(vc_dir, "vcvarsall.bat") - is_vs14 = True if os.path.isfile(vs14_vcvars) or vs_version == "14.0" else False - if is_vs14: - msvc_redist_dir = path.join(vc_dir, "redist", vs_platform, "Microsoft.VC140.CRT") - elif vs_version in ("15.0", "16.0"): - redist_dir = path.join(vc_dir, "Redist", "MSVC") - if os.path.isdir(redist_dir): - for p in os.listdir(redist_dir)[::-1]: - redist_path = path.join(redist_dir, p) - for v in ["VC141", "VC142", "VC150", "VC160"]: - # there are two possible paths - # `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT` - redist1 = path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v)) - redist2 = path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v)) - if os.path.isdir(redist1): - msvc_redist_dir = redist1 - break - elif os.path.isdir(redist2): - msvc_redist_dir = redist2 - break - if msvc_redist_dir: - break - if not msvc_redist_dir: - print("Couldn't locate MSVC redistributable directory") - return False - redist_dirs = [ - msvc_redist_dir, - ] - if "WindowsSdkDir" in os.environ: - redist_dirs += [path.join(os.environ["WindowsSdkDir"], "Redist", "ucrt", "DLLs", vs_platform)] missing = [] + redist_dirs = find_msvc_redist_dirs(target) for msvc_dll in msvc_deps: for dll_dir in redist_dirs: dll = path.join(dll_dir, msvc_dll) @@ -528,5 +457,5 @@ def package_msvc_dlls(servo_exe_dir, target, vcinstalldir, vs_version): missing += [msvc_dll] for msvc_dll in missing: - print("DLL file `{}` not found!".format(msvc_dll)) + print(f"Could not find DLL dependency: {msvc_dll}") return not missing diff --git a/python/servo/command_base.py b/python/servo/command_base.py index a6ea3add044..f6d105e7837 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -15,7 +15,6 @@ from typing import Dict, List, Optional import functools import gzip import itertools -import json import locale import os import platform @@ -477,23 +476,6 @@ class CommandBase(object): def msvc_package_dir(self, package): return servo.platform.windows.get_dependency_dir(package) - def vs_dirs(self): - assert 'windows' in servo.platform.host_triple() - vsinstalldir = os.environ.get('VSINSTALLDIR') - vs_version = os.environ.get('VisualStudioVersion') - if vsinstalldir and vs_version: - msbuild_version = get_msbuild_version(vs_version) - else: - (vsinstalldir, vs_version, msbuild_version) = find_highest_msvc_version() - msbuildinstalldir = os.path.join(vsinstalldir, "MSBuild", msbuild_version, "Bin") - vcinstalldir = os.environ.get("VCINSTALLDIR", "") or os.path.join(vsinstalldir, "VC") - return { - 'msbuild': msbuildinstalldir, - 'vsdir': vsinstalldir, - 'vs_version': vs_version, - 'vcdir': vcinstalldir, - } - def build_env(self): """Return an extended environment dictionary.""" env = os.environ.copy() @@ -1076,57 +1058,3 @@ class CommandBase(object): sys.exit(error) else: print("Clobber not needed.") - - -def find_highest_msvc_version_ext(): - def vswhere(args): - program_files = (os.environ.get('PROGRAMFILES(X86)') - or os.environ.get('PROGRAMFILES')) - if not program_files: - return [] - vswhere = os.path.join(program_files, 'Microsoft Visual Studio', - 'Installer', 'vswhere.exe') - if not os.path.exists(vswhere): - return [] - return json.loads(check_output([vswhere, '-format', 'json'] + args).decode(errors='ignore')) - - for install in vswhere(['-products', '*', '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', - '-requires', 'Microsoft.VisualStudio.Component.Windows10SDK']): - version = install['installationVersion'].split('.')[0] + '.0' - yield (install['installationPath'], version, "Current" if version == '16.0' else version) - - -def find_highest_msvc_version(): - editions = ["Enterprise", "Professional", "Community", "BuildTools"] - prog_files = os.environ.get("ProgramFiles(x86)") - base_vs_path = os.path.join(prog_files, "Microsoft Visual Studio") - - vs_versions = ["2019", "2017"] - versions = { - ("2019", "vs"): "16.0", - ("2017", "vs"): "15.0", - } - - for version in vs_versions: - for edition in editions: - vs_version = versions[version, "vs"] - msbuild_version = get_msbuild_version(vs_version) - - vsinstalldir = os.path.join(base_vs_path, version, edition) - if os.path.exists(vsinstalldir): - return (vsinstalldir, vs_version, msbuild_version) - - versions = sorted(find_highest_msvc_version_ext(), key=lambda tup: float(tup[1])) - if not versions: - print(f"Can't find MSBuild.exe installation under {base_vs_path}. " - "Please set the VSINSTALLDIR and VisualStudioVersion environment variables") - sys.exit(1) - return versions[0] - - -def get_msbuild_version(vs_version): - if vs_version in ("15.0", "14.0"): - msbuild_version = vs_version - else: - msbuild_version = "Current" - return msbuild_version diff --git a/python/servo/visual_studio.py b/python/servo/visual_studio.py new file mode 100644 index 00000000000..4149076a396 --- /dev/null +++ b/python/servo/visual_studio.py @@ -0,0 +1,149 @@ +# Copyright 2024 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import dataclasses +import json +import os +import subprocess +import sys +from typing import List + +import servo.platform + + +@dataclasses.dataclass(kw_only=True) +class VisualStudioInstallation: + version_number: str + installation_path: str + vc_install_path: str + + +def find_highest_msvc_version_ext(): + """Try to find the MSVC installation with the `vswhere.exe` tool. The results + are sorted with newer versions first.""" + def vswhere(args): + program_files = (os.environ.get('PROGRAMFILES(X86)') + or os.environ.get('PROGRAMFILES')) + if not program_files: + return [] + vswhere = os.path.join(program_files, 'Microsoft Visual Studio', 'Installer', 'vswhere.exe') + if not os.path.exists(vswhere): + return [] + output = subprocess.check_output([vswhere, '-format', 'json'] + args).decode(errors='ignore') + return json.loads(output) + + for install in vswhere(['-products', '*', + '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', + '-requires', 'Microsoft.VisualStudio.Component.Windows10SDK']): + version = install['installationVersion'].split('.')[0] + '.0' + yield (install['installationPath'], version) + + +def find_highest_msvc_version(): + prog_files = os.environ.get("ProgramFiles(x86)") + + # TODO(mrobinson): Add support for Visual Studio 2022. + vs_versions = { + "2019": "16.0", + } + + for (version, version_number) in vs_versions.items(): + for edition in ["Enterprise", "Professional", "Community", "BuildTools"]: + vsinstalldir = os.path.join(prog_files, "Microsoft Visual Studio", version, edition) + if os.path.exists(vsinstalldir): + return (vsinstalldir, version_number) + + versions = sorted(find_highest_msvc_version_ext(), key=lambda tup: float(tup[1])) + if not versions: + print("Can't find a Visual Studio installation. " + "Please set the VSINSTALLDIR and VisualStudioVersion environment variables") + sys.exit(1) + return versions[0] + + +def find_msvc() -> VisualStudioInstallation: + vsinstalldir = os.environ.get('VSINSTALLDIR') + version_number = os.environ.get('VisualStudioVersion') + if not vsinstalldir or not version_number: + (vsinstalldir, version_number) = find_highest_msvc_version() + + vc_install_path = os.environ.get("VCINSTALLDIR", os.path.join(vsinstalldir, "VC")) + if not os.path.exists(vc_install_path): + print(f"Can't find Visual C++ {version_number} installation at {vc_install_path}") + sys.exit(1) + + return VisualStudioInstallation( + version_number=version_number, + installation_path=vsinstalldir, + vc_install_path=vc_install_path, + ) + + +def find_windows_sdk_installation_path(vs_platform: str) -> str: + """Try to find the Windows SDK installation path using the Windows registry. + Raises an Exception if the path cannot be found in the registry.""" + + # This module must be imported here, because other platforms also + # load this file and the module is platform-specific. + import winreg + + # This is based on the advice from + # https://stackoverflow.com/questions/35119223/how-to-programmatically-detect-and-locate-the-windows-10-sdk + key_path = r'SOFTWARE\Wow6432Node\Microsoft\Microsoft SDKs\Windows\v10.0' + try: + with winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, key_path) as key: + path = str(winreg.QueryValueEx(key, "InstallationFolder")[0]) + return os.path.join(path, "Redist", "ucrt", "DLLs", vs_platform) + except FileNotFoundError: + raise Exception(f"Couldn't find Windows SDK installation path in registry at path ({key_path})") + + +def find_msvc_redist_dirs(target: str) -> List[str]: + assert 'windows' in servo.platform.host_triple() + + installation = find_msvc() + msvc_redist_dir = None + vs_platforms = { + "x86_64": "x64", + "i686": "x86", + "aarch64": "arm64", + } + target_arch = target.split('-')[0] + vs_platform = vs_platforms[target_arch] + + redist_dir = os.path.join(installation.vc_install_path, "Redist", "MSVC") + if not os.path.isdir(redist_dir): + raise Exception(f"Couldn't locate MSVC redistributable directory {redist_dir}") + + for p in os.listdir(redist_dir)[::-1]: + redist_path = os.path.join(redist_dir, p) + for v in ["VC141", "VC142", "VC150", "VC160"]: + # there are two possible paths + # `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT` + redist1 = os.path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v)) + redist2 = os.path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v)) + if os.path.isdir(redist1): + msvc_redist_dir = redist1 + break + elif os.path.isdir(redist2): + msvc_redist_dir = redist2 + break + if msvc_redist_dir: + break + + if not msvc_redist_dir: + print("Couldn't locate MSVC redistributable directory") + sys.exit(1) + + redist_dirs = [ + msvc_redist_dir, + find_windows_sdk_installation_path(vs_platform) + ] + + return redist_dirs