From f4d3d8ee1eeb94f643e113ff7fd52e043204c5ea Mon Sep 17 00:00:00 2001 From: UK992 Date: Tue, 16 Oct 2018 23:15:05 +0200 Subject: [PATCH 1/5] Windows: Add missing dependencies --- python/servo/build_commands.py | 100 +++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 4211cc84d96..42b09b59ced 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -18,6 +18,7 @@ import subprocess import sys import urllib import zipfile +import stat from time import time @@ -587,6 +588,105 @@ class MachCommands(CommandBase): for ssl_lib in ["libcryptoMD.dll", "libsslMD.dll"]: shutil.copy(path.join(env['OPENSSL_LIB_DIR'], "../bin" + msvc_x64, ssl_lib), servo_exe_dir) + # Search for the generated nspr4.dll + build_path = path.join(servo_exe_dir, "build") + nspr4 = "nspr4.dll" + nspr4_path = None + for root, dirs, files in os.walk(build_path): + if nspr4 in files: + nspr4_path = path.join(root, nspr4) + break + if nspr4_path is None: + print("WARNING: could not find nspr4.dll") + else: + shutil.copy(nspr4_path, servo_exe_dir) + # copy needed gstreamer DLLs in to servo.exe dir + gst_x64 = "X86_64" if msvc_x64 == "64" else "X86" + gst_root = "" + gst_default_path = path.join("C:\\gstreamer\\1.0", gst_x64) + gst_env = "GSTREAMER_1_0_ROOT_" + gst_x64 + if os.path.exists(path.join(gst_default_path, "bin", "libz.dll")): + gst_root = gst_default_path + elif os.environ.get(gst_env) is not None: + gst_root = os.environ.get(gst_env) + else: + print("Could not found GStreamer installation directory.") + status = 1 + gst_dlls = [ + "libz.dll", + "libintl-8.dll", + "liborc-0.4-0.dll", + "libwinpthread-1.dll", + "libffi-7.dll", + "libgobject-2.0-0.dll", + "libglib-2.0-0.dll", + "libgmodule-2.0-0.dll", + "libgstreamer-1.0-0.dll", + "libgstplayer-1.0-0.dll", + "libgstapp-1.0-0.dll", + "libgstaudio-1.0-0.dll", + "libgstvideo-1.0-0.dll", + "libgsttag-1.0-0.dll", + "libgstbase-1.0-0.dll", + "libgstpbutils-1.0-0.dll", + ] + if gst_root: + for gst_lib in gst_dlls: + shutil.copy(path.join(gst_root, "bin", gst_lib), + servo_exe_dir) + # copy some MSVC DLLs to servo.exe dir + msvc_redist_dir = None + vs_platform = os.environ.get("PLATFORM", "").lower() + vc_dir = os.environ.get("VCINSTALLDIR", "") + vs_version = os.environ.get("VisualStudioVersion", "") + msvc_deps = [ + "api-ms-win-crt-runtime-l1-1-0.dll", + "msvcp140.dll", + "vcruntime140.dll", + ] + # Check if it's Visual C++ Build Tools or Visual Studio 2015 + vs14_vcvars = path.join(vc_dir, "vcvarsall.bat") + is_vs14 = True if os.path.isfile(vs14_vcvars) or vs_version == "14.0" else False + if is_vs14: + msvc_redist_dir = path.join(vc_dir, "redist", vs_platform, "Microsoft.VC140.CRT") + elif vs_version == "15.0": + redist_dir = path.join(os.environ.get("VCINSTALLDIR", ""), "Redist", "MSVC") + if os.path.isdir(redist_dir): + for p in os.listdir(redist_dir)[::-1]: + redist_path = path.join(redist_dir, p) + for v in ["VC141", "VC150"]: + # there are two possible paths + # `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT` + redist1 = path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v)) + redist2 = path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v)) + if os.path.isdir(redist1): + msvc_redist_dir = redist1 + break + elif os.path.isdir(redist2): + msvc_redist_dir = redist2 + break + if msvc_redist_dir: + break + if msvc_redist_dir: + redist_dirs = [ + msvc_redist_dir, + path.join(os.environ["WindowsSdkDir"], "Redist", "ucrt", "DLLs", vs_platform), + ] + for msvc_dll in msvc_deps: + dll_found = False + for dll_dir in redist_dirs: + dll = path.join(dll_dir, msvc_dll) + servo_dir_dll = path.join(servo_exe_dir, msvc_dll) + if os.path.isfile(dll): + if os.path.isfile(servo_dir_dll): + # avoid permission denied error when overwrite dll in servo build directory + os.chmod(servo_dir_dll, stat.S_IWUSR) + shutil.copy(dll, servo_exe_dir) + dll_found = True + break + if not dll_found: + print("DLL file `{}` not found!".format(msvc_dll)) + status = 1 elif sys.platform == "darwin": # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools From c4b5c944c05a4ff30d0aa70b85dde6be3a47ec36 Mon Sep 17 00:00:00 2001 From: UK992 Date: Tue, 16 Oct 2018 23:16:00 +0200 Subject: [PATCH 2/5] Package: Update zip file creation --- python/servo/command_base.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 277f1d3aac1..2865eccd849 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -21,6 +21,7 @@ import subprocess from subprocess import PIPE import sys import tarfile +import zipfile from xml.etree.ElementTree import XML from servo.util import download_file import urllib2 @@ -85,10 +86,14 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None): dest_archive = os.path.abspath(dest_archive) with cd(dir_to_archive): current_dir = "." - file_list = [current_dir] + file_list = [] for root, dirs, files in os.walk(current_dir): - for name in itertools.chain(dirs, files): - file_list.append(os.path.join(root, name)) + if dest_archive.endswith(".zip"): + for f in files: + file_list.append(os.path.join(root, f)) + else: + for name in itertools.chain(dirs, files): + file_list.append(os.path.join(root, name)) # Sort file entries with the fixed locale with setlocale('C'): @@ -99,13 +104,21 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None): # TODO do this in a temporary folder after #11983 is fixed temp_file = '{}.temp~'.format(dest_archive) with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0644), 'w') as out_file: - with gzip.GzipFile('wb', fileobj=out_file, mtime=0) as gzip_file: - with tarfile.open(fileobj=gzip_file, mode='w:') as tar_file: + if dest_archive.endswith('.zip'): + with zipfile.ZipFile(temp_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: for entry in file_list: arcname = entry if prepend_path is not None: arcname = os.path.normpath(os.path.join(prepend_path, arcname)) - tar_file.add(entry, filter=reset, recursive=False, arcname=arcname) + zip_file.write(entry, arcname=arcname) + else: + with gzip.GzipFile('wb', fileobj=out_file, mtime=0) as gzip_file: + with tarfile.open(fileobj=gzip_file, mode='w:') as tar_file: + for entry in file_list: + arcname = entry + if prepend_path is not None: + arcname = os.path.normpath(os.path.join(prepend_path, arcname)) + tar_file.add(entry, filter=reset, recursive=False, arcname=arcname) os.rename(temp_file, dest_archive) From 06910456e296689706127d42334a5c99aca4d70a Mon Sep 17 00:00:00 2001 From: UK992 Date: Tue, 16 Oct 2018 23:53:53 +0200 Subject: [PATCH 3/5] Package: Update Windows packaging --- python/servo/package_commands.py | 37 +++++++++----------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 5449cfc509a..661e805a1c4 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -147,26 +147,9 @@ def copy_dependencies(binary_path, lib_path): def copy_windows_dependencies(binary_path, destination): - deps = [ - "libcryptoMD.dll", - "libsslMD.dll", - ] - for d in deps: - shutil.copy(path.join(binary_path, d), destination) - - # Search for the generated nspr4.dll - build_path = path.join(binary_path, "build") - nspr4 = "nspr4.dll" - nspr4_path = None - for root, dirs, files in os.walk(build_path): - if nspr4 in files: - nspr4_path = path.join(root, nspr4) - break - - if nspr4_path is None: - print("WARNING: could not find nspr4.dll") - else: - shutil.copy(nspr4_path, destination) + for f in os.listdir(binary_path): + if os.path.isfile(path.join(binary_path, f)) and f.endswith(".dll"): + shutil.copy(path.join(binary_path, f), destination) def change_prefs(resources_path, platform, vr=False): @@ -397,11 +380,10 @@ class PackageCommands(CommandBase): print("Copying files") dir_to_temp = path.join(dir_to_msi, 'temp') - dir_to_temp_servo = path.join(dir_to_temp, 'servo') - dir_to_resources = path.join(dir_to_temp_servo, 'resources') + dir_to_resources = path.join(dir_to_temp, 'resources') shutil.copytree(path.join(dir_to_root, 'resources'), dir_to_resources) - shutil.copy(binary_path, dir_to_temp_servo) - copy_windows_dependencies(target_dir, dir_to_temp_servo) + shutil.copy(binary_path, dir_to_temp) + copy_windows_dependencies(target_dir, dir_to_temp) change_prefs(dir_to_resources, "windows") @@ -412,7 +394,7 @@ class PackageCommands(CommandBase): wxs_path = path.join(dir_to_msi, "Installer.wxs") open(wxs_path, "w").write(template.render( exe_path=target_dir, - dir_to_temp=dir_to_temp_servo, + dir_to_temp=dir_to_temp, resources_path=dir_to_resources)) # run candle and light @@ -460,8 +442,9 @@ class PackageCommands(CommandBase): print("Packaged Servo into " + path.join(dir_to_msi, "Servo.msi")) print("Creating ZIP") - shutil.make_archive(path.join(dir_to_msi, "Servo"), "zip", dir_to_temp) - print("Packaged Servo into " + path.join(dir_to_msi, "Servo.zip")) + zip_path = path.join(dir_to_msi, "Servo.zip") + archive_deterministically(dir_to_temp, zip_path, prepend_path='servo/') + print("Packaged Servo into " + zip_path) print("Cleaning up") delete(dir_to_temp) From d9294282a9e8166a636446de6c7ca05fda5be5bb Mon Sep 17 00:00:00 2001 From: UK992 Date: Sat, 1 Dec 2018 23:01:20 +0100 Subject: [PATCH 4/5] Remove GStreamer installer from Servo bundle --- python/servo/package_commands.py | 12 ++---------- support/windows/Servo.wxs | 4 ---- support/windows/Servo.wxs.mako | 9 +-------- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 661e805a1c4..83f308f1816 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -19,7 +19,6 @@ import shutil import subprocess import sys import tempfile -import urllib from mach.decorators import ( CommandArgument, @@ -415,14 +414,7 @@ class PackageCommands(CommandBase): dir_to_installer = path.join(dir_to_msi, "Installer.msi") print("Packaged Servo into " + dir_to_installer) - # Download GStreamer installer. Only once. - gstreamer_msi_path = path.join(dir_to_msi, 'Gstreamer.msi') - if not os.path.exists(gstreamer_msi_path): - print('Fetching GStreamer installer. This may take a while...') - gstreamer_url = 'https://gstreamer.freedesktop.org/data/pkg/windows/1.14.2/gstreamer-1.0-x86-1.14.2.msi' - urllib.urlretrieve(gstreamer_url, gstreamer_msi_path) - - # Generate bundle with GStreamer and Servo installers. + # Generate bundle with Servo installer. print("Creating bundle") shutil.copy(path.join(dir_to_root, 'support', 'windows', 'Servo.wxs'), dir_to_msi) bundle_wxs_path = path.join(dir_to_msi, 'Servo.wxs') @@ -439,7 +431,7 @@ class PackageCommands(CommandBase): except subprocess.CalledProcessError as e: print("WiX light exited with return value %d" % e.returncode) return e.returncode - print("Packaged Servo into " + path.join(dir_to_msi, "Servo.msi")) + print("Packaged Servo into " + path.join(dir_to_msi, "Servo.exe")) print("Creating ZIP") zip_path = path.join(dir_to_msi, "Servo.zip") diff --git a/support/windows/Servo.wxs b/support/windows/Servo.wxs index d6ef3a04981..301e7108670 100644 --- a/support/windows/Servo.wxs +++ b/support/windows/Servo.wxs @@ -5,10 +5,6 @@ UpgradeCode="91b09c7e-6c0d-4166-b806-1dc724acf728"> - - - - - - - @@ -45,8 +39,7 @@ Icon="servo.exe" Advertise="yes"/> - ${include_dependencies()} - + ${include_dependencies()} ${include_directory(resources_path, "resources")} From f374dc3b7da5205bb1bf8d8d63b5e73f610309f0 Mon Sep 17 00:00:00 2001 From: UK992 Date: Fri, 1 Feb 2019 22:05:14 +0100 Subject: [PATCH 5/5] Taskcluster: Install gstreamer-1.0-x86_64-1.14.3.msi --- etc/taskcluster/decision_task.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index ea62d282021..17e12cba740 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -57,6 +57,7 @@ linux_build_env = { macos_build_env = {} windows_build_env = { "LIB": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\lib;%LIB%", + "GSTREAMER_1_0_ROOT_X86_64": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\", } windows_sparse_checkout = [ "/*", @@ -356,6 +357,12 @@ def windows_build_task(name): .with_repo(sparse_checkout=windows_sparse_checkout) .with_python2() .with_rustup() + .with_repacked_msi( + url="https://gstreamer.freedesktop.org/data/pkg/windows/" + + "1.14.3/gstreamer-1.0-x86_64-1.14.3.msi", + sha256="f4f20c713766ed6718b914b9ae57ed993a59ffe194e6ef530c8547508b4484d8", + path="gst", + ) .with_repacked_msi( url="https://gstreamer.freedesktop.org/data/pkg/windows/" + "1.14.3/gstreamer-1.0-devel-x86_64-1.14.3.msi",