build: Clean up post-build copy of Windows DLLs (#31092)

* build: Clean up post-build copy of Windows DLLs

- No longer use vcvarsall.bat at all. Instead find the Windows SDK
  directory by looking in the registry.
- Split logic for copying Windows dependencies into its own function and
  do some minor clean up, such as collecting all MSVC functionality into
  visual_studio.py.
- Remove support for Visual Studio 2015 and Visual Studio 2017.

This is a preparatory change in order to support Visual Studio 2022.

* More cleanup of the code

python/servo/build_commands.py

@@ -8,9 +8,9 @@
 # except according to those terms.
 
 import datetime
-import locale
 import os
 import os.path as path
+import pathlib
 import shutil
 import stat
 import subprocess
@@ -35,6 +35,7 @@ import servo.util
 
 from servo.command_base import BuildType, CommandBase, call, check_call
 from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins
+from python.servo.visual_studio import find_msvc_redist_dirs
 
 
 @CommandProvider
@@ -80,9 +81,6 @@ class MachCommands(CommandBase):
         build_start = time()
 
         host = servo.platform.host_triple()
-        if 'windows' in host:
-            vs_dirs = self.vs_dirs()
-
         target_triple = self.cross_compile_target or servo.platform.host_triple()
         if host != target_triple and 'windows' in target_triple:
             if os.environ.get('VisualStudioVersion') or os.environ.get('VCINSTALLDIR'):
@@ -91,29 +89,6 @@ class MachCommands(CommandBase):
                       "Visual Studio shell, and make sure the VisualStudioVersion and "
                       "VCINSTALLDIR environment variables are not set.")
                 sys.exit(1)
-            vcinstalldir = vs_dirs['vcdir']
-            if not os.path.exists(vcinstalldir):
-                print("Can't find Visual C++ %s installation at %s." % (vs_dirs['vs_version'], vcinstalldir))
-                sys.exit(1)
-
-            env['PKG_CONFIG_ALLOW_CROSS'] = "1"
-
-        if 'windows' in host:
-            process = subprocess.Popen('("%s" %s > nul) && "python" -c "import os; print(repr(os.environ))"' %
-                                       (os.path.join(vs_dirs['vcdir'], "Auxiliary", "Build", "vcvarsall.bat"), "x64"),
-                                       stdout=subprocess.PIPE, shell=True)
-            stdout, stderr = process.communicate()
-            exitcode = process.wait()
-            encoding = locale.getpreferredencoding()  # See https://stackoverflow.com/a/9228117
-            if exitcode == 0:
-                decoded = stdout.decode(encoding)
-                if decoded.startswith("environ("):
-                    decoded = decoded.strip()[8:-1]
-                os.environ.update(eval(decoded))
-            else:
-                print("Failed to run vcvarsall. stderr:")
-                print(stderr.decode(encoding))
-                exit(1)
 
         # Gather Cargo build timings (https://doc.rust-lang.org/cargo/reference/timings.html).
         opts = ["--timings"] + opts
@@ -130,6 +105,7 @@ class MachCommands(CommandBase):
         )
 
         # Do some additional things if the build succeeded
+        built_binary = self.get_binary_path(build_type, target=self.cross_compile_target, simpleservo=libsimpleservo)
         if status == 0:
             if self.is_android_build and not no_package:
                 flavor = None
@@ -143,43 +119,7 @@ class MachCommands(CommandBase):
                     return rv
 
             if sys.platform == "win32":
-                servo_exe_dir = os.path.dirname(
+                if not copy_windows_dlls_to_build_directory(built_binary, target_triple):
-                    self.get_binary_path(build_type, target=self.cross_compile_target, simpleservo=libsimpleservo)
-                )
-                assert os.path.exists(servo_exe_dir)
-
-                build_path = path.join(servo_exe_dir, "build")
-                assert os.path.exists(build_path)
-
-                # on msvc, we need to copy in some DLLs in to the servo.exe dir and the directory for unit tests.
-                def package_generated_shared_libraries(libs, build_path, servo_exe_dir):
-                    for root, dirs, files in os.walk(build_path):
-                        remaining_libs = list(libs)
-                        for lib in libs:
-                            if lib in files:
-                                shutil.copy(path.join(root, lib), servo_exe_dir)
-                                remaining_libs.remove(lib)
-                                continue
-                        libs = remaining_libs
-                        if not libs:
-                            return True
-                    for lib in libs:
-                        print("WARNING: could not find " + lib)
-
-                print("Packaging EGL DLLs")
-                egl_libs = ["libEGL.dll", "libGLESv2.dll"]
-                if not package_generated_shared_libraries(egl_libs, build_path, servo_exe_dir):
-                    status = 1
-
-                # copy needed gstreamer DLLs in to servo.exe dir
-                if self.enable_media:
-                    print("Packaging gstreamer DLLs")
-                    if not package_gstreamer_dlls(env, servo_exe_dir, target_triple):
-                        status = 1
-
-                # UWP app packaging already bundles all required DLLs for us.
-                print("Packaging MSVC DLLs")
-                if not package_msvc_dlls(servo_exe_dir, target_triple, vs_dirs['vcdir'], vs_dirs['vs_version']):
                     status = 1
 
             elif sys.platform == "darwin":
@@ -423,7 +363,38 @@ def package_gstreamer_dylibs(cross_compilation_target, servo_bin):
     return True
 
 
-def package_gstreamer_dlls(env, servo_exe_dir, target):
+def copy_windows_dlls_to_build_directory(servo_binary: str, target_triple: str) -> bool:
+    servo_exe_dir = os.path.dirname(servo_binary)
+    assert os.path.exists(servo_exe_dir)
+
+    build_path = path.join(servo_exe_dir, "build")
+    assert os.path.exists(build_path)
+
+    # Copy in the built EGL and GLES libraries from where they were built to
+    # the final build dirctory
+    def find_and_copy_built_dll(dll_name):
+        try:
+            file_to_copy = next(pathlib.Path(build_path).rglob(dll_name))
+            shutil.copy(file_to_copy, servo_exe_dir)
+        except StopIteration:
+            print(f"WARNING: could not find {dll_name}")
+
+    print(" • Copying ANGLE DLLs to binary directory...")
+    find_and_copy_built_dll("libEGL.dll")
+    find_and_copy_built_dll("libGLESv2.dll")
+
+    print(" • Copying GStreamer DLLs to binary directory...")
+    if not package_gstreamer_dlls(servo_exe_dir, target_triple):
+        return False
+
+    print(" • Copying MSVC DLLs to binary directory...")
+    if not package_msvc_dlls(servo_exe_dir, target_triple):
+        return False
+
+    return True
+
+
+def package_gstreamer_dlls(servo_exe_dir: str, target: str):
     gst_root = servo.platform.get().gstreamer_root(cross_compilation_target=target)
     if not gst_root:
         print("Could not find GStreamer installation directory.")
@@ -462,58 +433,16 @@ def package_gstreamer_dlls(env, servo_exe_dir, target):
     return not missing
 
 
-def package_msvc_dlls(servo_exe_dir, target, vcinstalldir, vs_version):
+def package_msvc_dlls(servo_exe_dir, target):
-    # copy some MSVC DLLs to servo.exe dir
-    msvc_redist_dir = None
-    vs_platforms = {
-        "x86_64": "x64",
-        "i686": "x86",
-        "aarch64": "arm64",
-    }
-    target_arch = target.split('-')[0]
-    vs_platform = vs_platforms[target_arch]
-    vc_dir = vcinstalldir or os.environ.get("VCINSTALLDIR", "")
-    if not vs_version:
-        vs_version = os.environ.get("VisualStudioVersion", "")
     msvc_deps = [
         "msvcp140.dll",
         "vcruntime140.dll",
     ]
-    if target_arch != "aarch64" and vs_version in ("14.0", "15.0", "16.0"):
+    if "aarch64" not in target != "aarch64":
         msvc_deps += ["api-ms-win-crt-runtime-l1-1-0.dll"]
 
-    # Check if it's Visual C++ Build Tools or Visual Studio 2015
-    vs14_vcvars = path.join(vc_dir, "vcvarsall.bat")
-    is_vs14 = True if os.path.isfile(vs14_vcvars) or vs_version == "14.0" else False
-    if is_vs14:
-        msvc_redist_dir = path.join(vc_dir, "redist", vs_platform, "Microsoft.VC140.CRT")
-    elif vs_version in ("15.0", "16.0"):
-        redist_dir = path.join(vc_dir, "Redist", "MSVC")
-        if os.path.isdir(redist_dir):
-            for p in os.listdir(redist_dir)[::-1]:
-                redist_path = path.join(redist_dir, p)
-                for v in ["VC141", "VC142", "VC150", "VC160"]:
-                    # there are two possible paths
-                    # `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT`
-                    redist1 = path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v))
-                    redist2 = path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v))
-                    if os.path.isdir(redist1):
-                        msvc_redist_dir = redist1
-                        break
-                    elif os.path.isdir(redist2):
-                        msvc_redist_dir = redist2
-                        break
-                if msvc_redist_dir:
-                    break
-    if not msvc_redist_dir:
-        print("Couldn't locate MSVC redistributable directory")
-        return False
-    redist_dirs = [
-        msvc_redist_dir,
-    ]
-    if "WindowsSdkDir" in os.environ:
-        redist_dirs += [path.join(os.environ["WindowsSdkDir"], "Redist", "ucrt", "DLLs", vs_platform)]
     missing = []
+    redist_dirs = find_msvc_redist_dirs(target)
     for msvc_dll in msvc_deps:
         for dll_dir in redist_dirs:
             dll = path.join(dll_dir, msvc_dll)
@@ -528,5 +457,5 @@ def package_msvc_dlls(servo_exe_dir, target, vcinstalldir, vs_version):
             missing += [msvc_dll]
 
     for msvc_dll in missing:
-        print("DLL file `{}` not found!".format(msvc_dll))
+        print(f"Could not find DLL dependency: {msvc_dll}")
     return not missing

python/servo/command_base.py

@@ -15,7 +15,6 @@ from typing import Dict, List, Optional
 import functools
 import gzip
 import itertools
-import json
 import locale
 import os
 import platform
@@ -477,23 +476,6 @@ class CommandBase(object):
     def msvc_package_dir(self, package):
         return servo.platform.windows.get_dependency_dir(package)
 
-    def vs_dirs(self):
-        assert 'windows' in servo.platform.host_triple()
-        vsinstalldir = os.environ.get('VSINSTALLDIR')
-        vs_version = os.environ.get('VisualStudioVersion')
-        if vsinstalldir and vs_version:
-            msbuild_version = get_msbuild_version(vs_version)
-        else:
-            (vsinstalldir, vs_version, msbuild_version) = find_highest_msvc_version()
-        msbuildinstalldir = os.path.join(vsinstalldir, "MSBuild", msbuild_version, "Bin")
-        vcinstalldir = os.environ.get("VCINSTALLDIR", "") or os.path.join(vsinstalldir, "VC")
-        return {
-            'msbuild': msbuildinstalldir,
-            'vsdir': vsinstalldir,
-            'vs_version': vs_version,
-            'vcdir': vcinstalldir,
-        }
-
     def build_env(self):
         """Return an extended environment dictionary."""
         env = os.environ.copy()
@@ -1076,57 +1058,3 @@ class CommandBase(object):
                     sys.exit(error)
             else:
                 print("Clobber not needed.")
-
-
-def find_highest_msvc_version_ext():
-    def vswhere(args):
-        program_files = (os.environ.get('PROGRAMFILES(X86)')
-                         or os.environ.get('PROGRAMFILES'))
-        if not program_files:
-            return []
-        vswhere = os.path.join(program_files, 'Microsoft Visual Studio',
-                               'Installer', 'vswhere.exe')
-        if not os.path.exists(vswhere):
-            return []
-        return json.loads(check_output([vswhere, '-format', 'json'] + args).decode(errors='ignore'))
-
-    for install in vswhere(['-products', '*', '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
-                            '-requires', 'Microsoft.VisualStudio.Component.Windows10SDK']):
-        version = install['installationVersion'].split('.')[0] + '.0'
-        yield (install['installationPath'], version, "Current" if version == '16.0' else version)
-
-
-def find_highest_msvc_version():
-    editions = ["Enterprise", "Professional", "Community", "BuildTools"]
-    prog_files = os.environ.get("ProgramFiles(x86)")
-    base_vs_path = os.path.join(prog_files, "Microsoft Visual Studio")
-
-    vs_versions = ["2019", "2017"]
-    versions = {
-        ("2019", "vs"): "16.0",
-        ("2017", "vs"): "15.0",
-    }
-
-    for version in vs_versions:
-        for edition in editions:
-            vs_version = versions[version, "vs"]
-            msbuild_version = get_msbuild_version(vs_version)
-
-            vsinstalldir = os.path.join(base_vs_path, version, edition)
-            if os.path.exists(vsinstalldir):
-                return (vsinstalldir, vs_version, msbuild_version)
-
-    versions = sorted(find_highest_msvc_version_ext(), key=lambda tup: float(tup[1]))
-    if not versions:
-        print(f"Can't find MSBuild.exe installation under {base_vs_path}. "
-              "Please set the VSINSTALLDIR and VisualStudioVersion environment variables")
-        sys.exit(1)
-    return versions[0]
-
-
-def get_msbuild_version(vs_version):
-    if vs_version in ("15.0", "14.0"):
-        msbuild_version = vs_version
-    else:
-        msbuild_version = "Current"
-    return msbuild_version

python/servo/visual_studio.py

@@ -0,0 +1,149 @@
+# Copyright 2024 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import dataclasses
+import json
+import os
+import subprocess
+import sys
+from typing import List
+
+import servo.platform
+
+
+@dataclasses.dataclass(kw_only=True)
+class VisualStudioInstallation:
+    version_number: str
+    installation_path: str
+    vc_install_path: str
+
+
+def find_highest_msvc_version_ext():
+    """Try to find the MSVC installation with the `vswhere.exe` tool. The results
+    are sorted with newer versions first."""
+    def vswhere(args):
+        program_files = (os.environ.get('PROGRAMFILES(X86)')
+                         or os.environ.get('PROGRAMFILES'))
+        if not program_files:
+            return []
+        vswhere = os.path.join(program_files, 'Microsoft Visual Studio', 'Installer', 'vswhere.exe')
+        if not os.path.exists(vswhere):
+            return []
+        output = subprocess.check_output([vswhere, '-format', 'json'] + args).decode(errors='ignore')
+        return json.loads(output)
+
+    for install in vswhere(['-products', '*',
+                            '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
+                            '-requires', 'Microsoft.VisualStudio.Component.Windows10SDK']):
+        version = install['installationVersion'].split('.')[0] + '.0'
+        yield (install['installationPath'], version)
+
+
+def find_highest_msvc_version():
+    prog_files = os.environ.get("ProgramFiles(x86)")
+
+    # TODO(mrobinson): Add support for Visual Studio 2022.
+    vs_versions = {
+        "2019": "16.0",
+    }
+
+    for (version, version_number) in vs_versions.items():
+        for edition in ["Enterprise", "Professional", "Community", "BuildTools"]:
+            vsinstalldir = os.path.join(prog_files, "Microsoft Visual Studio", version, edition)
+            if os.path.exists(vsinstalldir):
+                return (vsinstalldir, version_number)
+
+    versions = sorted(find_highest_msvc_version_ext(), key=lambda tup: float(tup[1]))
+    if not versions:
+        print("Can't find a Visual Studio installation. "
+              "Please set the VSINSTALLDIR and VisualStudioVersion environment variables")
+        sys.exit(1)
+    return versions[0]
+
+
+def find_msvc() -> VisualStudioInstallation:
+    vsinstalldir = os.environ.get('VSINSTALLDIR')
+    version_number = os.environ.get('VisualStudioVersion')
+    if not vsinstalldir or not version_number:
+        (vsinstalldir, version_number) = find_highest_msvc_version()
+
+    vc_install_path = os.environ.get("VCINSTALLDIR", os.path.join(vsinstalldir, "VC"))
+    if not os.path.exists(vc_install_path):
+        print(f"Can't find Visual C++ {version_number} installation at {vc_install_path}")
+        sys.exit(1)
+
+    return VisualStudioInstallation(
+        version_number=version_number,
+        installation_path=vsinstalldir,
+        vc_install_path=vc_install_path,
+    )
+
+
+def find_windows_sdk_installation_path(vs_platform: str) -> str:
+    """Try to find the Windows SDK installation path using the Windows registry.
+    Raises an Exception if the path cannot be found in the registry."""
+
+    # This module must be imported here, because other platforms also
+    # load this file and the module is platform-specific.
+    import winreg
+
+    # This is based on the advice from
+    # https://stackoverflow.com/questions/35119223/how-to-programmatically-detect-and-locate-the-windows-10-sdk
+    key_path = r'SOFTWARE\Wow6432Node\Microsoft\Microsoft SDKs\Windows\v10.0'
+    try:
+        with winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, key_path) as key:
+            path = str(winreg.QueryValueEx(key, "InstallationFolder")[0])
+            return os.path.join(path, "Redist", "ucrt", "DLLs", vs_platform)
+    except FileNotFoundError:
+        raise Exception(f"Couldn't find Windows SDK installation path in registry at path ({key_path})")
+
+
+def find_msvc_redist_dirs(target: str) -> List[str]:
+    assert 'windows' in servo.platform.host_triple()
+
+    installation = find_msvc()
+    msvc_redist_dir = None
+    vs_platforms = {
+        "x86_64": "x64",
+        "i686": "x86",
+        "aarch64": "arm64",
+    }
+    target_arch = target.split('-')[0]
+    vs_platform = vs_platforms[target_arch]
+
+    redist_dir = os.path.join(installation.vc_install_path, "Redist", "MSVC")
+    if not os.path.isdir(redist_dir):
+        raise Exception(f"Couldn't locate MSVC redistributable directory {redist_dir}")
+
+    for p in os.listdir(redist_dir)[::-1]:
+        redist_path = os.path.join(redist_dir, p)
+        for v in ["VC141", "VC142", "VC150", "VC160"]:
+            # there are two possible paths
+            # `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT`
+            redist1 = os.path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v))
+            redist2 = os.path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v))
+            if os.path.isdir(redist1):
+                msvc_redist_dir = redist1
+                break
+            elif os.path.isdir(redist2):
+                msvc_redist_dir = redist2
+                break
+        if msvc_redist_dir:
+            break
+
+    if not msvc_redist_dir:
+        print("Couldn't locate MSVC redistributable directory")
+        sys.exit(1)
+
+    redist_dirs = [
+        msvc_redist_dir,
+        find_windows_sdk_installation_path(vs_platform)
+    ]
+
+    return redist_dirs