Auto merge of #21968 - UK992:mach-package, r=jdm

Windows: Add missing dependencies

Rebased https://github.com/servo/servo/pull/16445 and updated with Gstreamer DLLs.

About msi installer, there is also included gstreamer installer, should be removed and replaced by needed gstreamer DLLs or keep it at is it?

Fixes #16422.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/21968)
<!-- Reviewable:end -->

etc/taskcluster/decision_task.py

@@ -120,6 +120,7 @@ linux_build_env = {
 macos_build_env = {}
 windows_build_env = {
     "LIB": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\lib;%LIB%",
+    "GSTREAMER_1_0_ROOT_X86_64": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\",
 }
 windows_sparse_checkout = [
     "/*",
@@ -562,6 +563,12 @@ def windows_build_task(name):
         .with_repo(sparse_checkout=windows_sparse_checkout)
         .with_python2()
         .with_rustup()
+        .with_repacked_msi(
+            url="https://gstreamer.freedesktop.org/data/pkg/windows/" +
+                "1.14.3/gstreamer-1.0-x86_64-1.14.3.msi",
+            sha256="f4f20c713766ed6718b914b9ae57ed993a59ffe194e6ef530c8547508b4484d8",
+            path="gst",
+        )
         .with_repacked_msi(
             url="https://gstreamer.freedesktop.org/data/pkg/windows/" +
                 "1.14.3/gstreamer-1.0-devel-x86_64-1.14.3.msi",

python/servo/build_commands.py

@@ -18,6 +18,7 @@ import subprocess
 import sys
 import urllib
 import zipfile
+import stat
 
 from time import time
 
@@ -592,6 +593,105 @@ class MachCommands(CommandBase):
                 for ssl_lib in ["libcryptoMD.dll", "libsslMD.dll"]:
                     shutil.copy(path.join(env['OPENSSL_LIB_DIR'], "../bin" + msvc_x64, ssl_lib),
                                 servo_exe_dir)
+                # Search for the generated nspr4.dll
+                build_path = path.join(servo_exe_dir, "build")
+                nspr4 = "nspr4.dll"
+                nspr4_path = None
+                for root, dirs, files in os.walk(build_path):
+                    if nspr4 in files:
+                        nspr4_path = path.join(root, nspr4)
+                        break
+                if nspr4_path is None:
+                    print("WARNING: could not find nspr4.dll")
+                else:
+                    shutil.copy(nspr4_path, servo_exe_dir)
+                # copy needed gstreamer DLLs in to servo.exe dir
+                gst_x64 = "X86_64" if msvc_x64 == "64" else "X86"
+                gst_root = ""
+                gst_default_path = path.join("C:\\gstreamer\\1.0", gst_x64)
+                gst_env = "GSTREAMER_1_0_ROOT_" + gst_x64
+                if os.path.exists(path.join(gst_default_path, "bin", "libz.dll")):
+                    gst_root = gst_default_path
+                elif os.environ.get(gst_env) is not None:
+                    gst_root = os.environ.get(gst_env)
+                else:
+                    print("Could not found GStreamer installation directory.")
+                    status = 1
+                gst_dlls = [
+                    "libz.dll",
+                    "libintl-8.dll",
+                    "liborc-0.4-0.dll",
+                    "libwinpthread-1.dll",
+                    "libffi-7.dll",
+                    "libgobject-2.0-0.dll",
+                    "libglib-2.0-0.dll",
+                    "libgmodule-2.0-0.dll",
+                    "libgstreamer-1.0-0.dll",
+                    "libgstplayer-1.0-0.dll",
+                    "libgstapp-1.0-0.dll",
+                    "libgstaudio-1.0-0.dll",
+                    "libgstvideo-1.0-0.dll",
+                    "libgsttag-1.0-0.dll",
+                    "libgstbase-1.0-0.dll",
+                    "libgstpbutils-1.0-0.dll",
+                ]
+                if gst_root:
+                    for gst_lib in gst_dlls:
+                        shutil.copy(path.join(gst_root, "bin", gst_lib),
+                                    servo_exe_dir)
+                # copy some MSVC DLLs to servo.exe dir
+                msvc_redist_dir = None
+                vs_platform = os.environ.get("PLATFORM", "").lower()
+                vc_dir = os.environ.get("VCINSTALLDIR", "")
+                vs_version = os.environ.get("VisualStudioVersion", "")
+                msvc_deps = [
+                    "api-ms-win-crt-runtime-l1-1-0.dll",
+                    "msvcp140.dll",
+                    "vcruntime140.dll",
+                ]
+                # Check if it's Visual C++ Build Tools or Visual Studio 2015
+                vs14_vcvars = path.join(vc_dir, "vcvarsall.bat")
+                is_vs14 = True if os.path.isfile(vs14_vcvars) or vs_version == "14.0" else False
+                if is_vs14:
+                    msvc_redist_dir = path.join(vc_dir, "redist", vs_platform, "Microsoft.VC140.CRT")
+                elif vs_version == "15.0":
+                    redist_dir = path.join(os.environ.get("VCINSTALLDIR", ""), "Redist", "MSVC")
+                    if os.path.isdir(redist_dir):
+                        for p in os.listdir(redist_dir)[::-1]:
+                            redist_path = path.join(redist_dir, p)
+                            for v in ["VC141", "VC150"]:
+                                # there are two possible paths
+                                # `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT`
+                                redist1 = path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v))
+                                redist2 = path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v))
+                                if os.path.isdir(redist1):
+                                    msvc_redist_dir = redist1
+                                    break
+                                elif os.path.isdir(redist2):
+                                    msvc_redist_dir = redist2
+                                    break
+                            if msvc_redist_dir:
+                                break
+                if msvc_redist_dir:
+                    redist_dirs = [
+                        msvc_redist_dir,
+                        path.join(os.environ["WindowsSdkDir"], "Redist", "ucrt", "DLLs", vs_platform),
+                    ]
+                    for msvc_dll in msvc_deps:
+                        dll_found = False
+                        for dll_dir in redist_dirs:
+                            dll = path.join(dll_dir, msvc_dll)
+                            servo_dir_dll = path.join(servo_exe_dir, msvc_dll)
+                            if os.path.isfile(dll):
+                                if os.path.isfile(servo_dir_dll):
+                                    # avoid permission denied error when overwrite dll in servo build directory
+                                    os.chmod(servo_dir_dll, stat.S_IWUSR)
+                                shutil.copy(dll, servo_exe_dir)
+                                dll_found = True
+                                break
+                        if not dll_found:
+                            print("DLL file `{}` not found!".format(msvc_dll))
+                            status = 1
 
             elif sys.platform == "darwin":
                 # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools

python/servo/command_base.py

@@ -22,6 +22,7 @@ import subprocess
 from subprocess import PIPE
 import sys
 import tarfile
+import zipfile
 from xml.etree.ElementTree import XML
 from servo.util import download_file
 import urllib2
@@ -86,8 +87,12 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
     dest_archive = os.path.abspath(dest_archive)
     with cd(dir_to_archive):
         current_dir = "."
-        file_list = [current_dir]
+        file_list = []
         for root, dirs, files in os.walk(current_dir):
+            if dest_archive.endswith(".zip"):
+                for f in files:
+                    file_list.append(os.path.join(root, f))
+            else:
                 for name in itertools.chain(dirs, files):
                     file_list.append(os.path.join(root, name))
 
@@ -100,6 +105,14 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
         # TODO do this in a temporary folder after #11983 is fixed
         temp_file = '{}.temp~'.format(dest_archive)
         with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0644), 'w') as out_file:
+            if dest_archive.endswith('.zip'):
+                with zipfile.ZipFile(temp_file, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+                    for entry in file_list:
+                        arcname = entry
+                        if prepend_path is not None:
+                            arcname = os.path.normpath(os.path.join(prepend_path, arcname))
+                        zip_file.write(entry, arcname=arcname)
+            else:
                 with gzip.GzipFile('wb', fileobj=out_file, mtime=0) as gzip_file:
                     with tarfile.open(fileobj=gzip_file, mode='w:') as tar_file:
                         for entry in file_list:

python/servo/package_commands.py

@@ -19,7 +19,6 @@ import shutil
 import subprocess
 import sys
 import tempfile
-import urllib
 
 from mach.decorators import (
     CommandArgument,
@@ -147,26 +146,9 @@ def copy_dependencies(binary_path, lib_path):
 
 
 def copy_windows_dependencies(binary_path, destination):
-    deps = [
+    for f in os.listdir(binary_path):
-        "libcryptoMD.dll",
+        if os.path.isfile(path.join(binary_path, f)) and f.endswith(".dll"):
-        "libsslMD.dll",
+            shutil.copy(path.join(binary_path, f), destination)
-    ]
-    for d in deps:
-        shutil.copy(path.join(binary_path, d), destination)
-
-    # Search for the generated nspr4.dll
-    build_path = path.join(binary_path, "build")
-    nspr4 = "nspr4.dll"
-    nspr4_path = None
-    for root, dirs, files in os.walk(build_path):
-        if nspr4 in files:
-            nspr4_path = path.join(root, nspr4)
-            break
-
-    if nspr4_path is None:
-        print("WARNING: could not find nspr4.dll")
-    else:
-        shutil.copy(nspr4_path, destination)
 
 
 def change_prefs(resources_path, platform, vr=False):
@@ -397,11 +379,10 @@ class PackageCommands(CommandBase):
 
             print("Copying files")
             dir_to_temp = path.join(dir_to_msi, 'temp')
-            dir_to_temp_servo = path.join(dir_to_temp, 'servo')
+            dir_to_resources = path.join(dir_to_temp, 'resources')
-            dir_to_resources = path.join(dir_to_temp_servo, 'resources')
             shutil.copytree(path.join(dir_to_root, 'resources'), dir_to_resources)
-            shutil.copy(binary_path, dir_to_temp_servo)
+            shutil.copy(binary_path, dir_to_temp)
-            copy_windows_dependencies(target_dir, dir_to_temp_servo)
+            copy_windows_dependencies(target_dir, dir_to_temp)
 
             change_prefs(dir_to_resources, "windows")
 
@@ -412,7 +393,7 @@ class PackageCommands(CommandBase):
             wxs_path = path.join(dir_to_msi, "Installer.wxs")
             open(wxs_path, "w").write(template.render(
                 exe_path=target_dir,
-                dir_to_temp=dir_to_temp_servo,
+                dir_to_temp=dir_to_temp,
                 resources_path=dir_to_resources))
 
             # run candle and light
@@ -433,14 +414,7 @@ class PackageCommands(CommandBase):
             dir_to_installer = path.join(dir_to_msi, "Installer.msi")
             print("Packaged Servo into " + dir_to_installer)
 
-            # Download GStreamer installer. Only once.
+            # Generate bundle with Servo installer.
-            gstreamer_msi_path = path.join(dir_to_msi, 'Gstreamer.msi')
-            if not os.path.exists(gstreamer_msi_path):
-                print('Fetching GStreamer installer. This may take a while...')
-                gstreamer_url = 'https://gstreamer.freedesktop.org/data/pkg/windows/1.14.2/gstreamer-1.0-x86-1.14.2.msi'
-                urllib.urlretrieve(gstreamer_url, gstreamer_msi_path)
-
-            # Generate bundle with GStreamer and Servo installers.
             print("Creating bundle")
             shutil.copy(path.join(dir_to_root, 'support', 'windows', 'Servo.wxs'), dir_to_msi)
             bundle_wxs_path = path.join(dir_to_msi, 'Servo.wxs')
@@ -457,11 +431,12 @@ class PackageCommands(CommandBase):
             except subprocess.CalledProcessError as e:
                 print("WiX light exited with return value %d" % e.returncode)
                 return e.returncode
-            print("Packaged Servo into " + path.join(dir_to_msi, "Servo.msi"))
+            print("Packaged Servo into " + path.join(dir_to_msi, "Servo.exe"))
 
             print("Creating ZIP")
-            shutil.make_archive(path.join(dir_to_msi, "Servo"), "zip", dir_to_temp)
+            zip_path = path.join(dir_to_msi, "Servo.zip")
-            print("Packaged Servo into " + path.join(dir_to_msi, "Servo.zip"))
+            archive_deterministically(dir_to_temp, zip_path, prepend_path='servo/')
+            print("Packaged Servo into " + zip_path)
 
             print("Cleaning up")
             delete(dir_to_temp)

support/windows/Servo.wxs

@@ -5,10 +5,6 @@
           UpgradeCode="91b09c7e-6c0d-4166-b806-1dc724acf728">
     <BootstrapperApplicationRef Id="WixStandardBootstrapperApplication.RtfLicense" />
     <Chain>
-      <MsiPackage
-          SourceFile="Gstreamer.msi"
-          DisplayName="GStreamer"
-          ForcePerMachine="yes"/>
       <MsiPackage
           SourceFile="Installer.msi"
           Compressed="yes"

support/windows/Servo.wxs.mako

@@ -20,12 +20,6 @@
     <Media Id="1"
            Cabinet="Servo.cab"
            EmbedCab="yes"/>
-    <Property Id="GSTINSTALLDIR">
-	    <RegistrySearch Id="GStreamerRegistry" Type="raw" Root="HKLM" Key="SOFTWARE\WOW6432Node\GStreamer1.0\x86" Name="InstallDir"/>
-    </Property>
-    <Property Id="GSTINSTALLDIR64">
-	    <RegistrySearch Id="GStreamerRegistry64" Type="raw" Root="HKLM" Key="SOFTWARE\WOW6432Node\GStreamer1.0\x86_64" Name="InstallDir"/>
-    </Property>
     <Directory Id="TARGETDIR" Name="SourceDir">
       <Directory Id="ProgramFiles64Folder" Name="PFiles">
         <Directory Id="MozResearch" Name="Mozilla Research">
@@ -46,7 +40,6 @@
                           Advertise="yes"/>
               </File>
 	            ${include_dependencies()}
-              <Environment Id="PATH" Name="PATH" Value="[GSTINSTALLDIR]1.0\x86\bin;[GSTINSTALLDIR64]1.0\x86_64\bin" Permanent="yes" Part="last" Action="set" System="yes" />
             </Component>
 
             ${include_directory(resources_path, "resources")}