mirror of
https://github.com/servo/servo.git
synced 2025-07-26 08:40:24 +01:00
54dd8becc1
Add documentation and simplify the way that GStreamer shared objects lists are stored. In addition, move the list of extra GStreamer DLL dependencies to to the `gstreamer.py` file. The conditional plugin logic is no longer required as we are having already increased our GStreamer version requirements.
532 lines
22 KiB
Python
532 lines
22 KiB
Python
# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
|
|
# file at the top-level directory of this distribution.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
# option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
import datetime
|
|
import locale
|
|
import os
|
|
import os.path as path
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import urllib
|
|
|
|
from time import time
|
|
from typing import Dict
|
|
import zipfile
|
|
|
|
import notifypy
|
|
|
|
from mach.decorators import (
|
|
CommandArgument,
|
|
CommandProvider,
|
|
Command,
|
|
)
|
|
from mach.registrar import Registrar
|
|
|
|
import servo.platform
|
|
import servo.util
|
|
|
|
from servo.command_base import BuildType, CommandBase, call, check_call
|
|
from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins
|
|
|
|
|
|
@CommandProvider
|
|
class MachCommands(CommandBase):
|
|
@Command('build', description='Build Servo', category='build')
|
|
@CommandArgument('--jobs', '-j',
|
|
default=None,
|
|
help='Number of jobs to run in parallel')
|
|
@CommandArgument('--no-package',
|
|
action='store_true',
|
|
help='For Android, disable packaging into a .apk after building')
|
|
@CommandArgument('--verbose', '-v',
|
|
action='store_true',
|
|
help='Print verbose output')
|
|
@CommandArgument('--very-verbose', '-vv',
|
|
action='store_true',
|
|
help='Print very verbose output')
|
|
@CommandArgument('params', nargs='...',
|
|
help="Command-line arguments to be passed through to Cargo")
|
|
@CommandBase.common_command_arguments(build_configuration=True, build_type=True)
|
|
def build(self, build_type: BuildType, jobs=None, params=None, no_package=False,
|
|
verbose=False, very_verbose=False, libsimpleservo=False, **kwargs):
|
|
opts = params or []
|
|
|
|
if build_type.is_release():
|
|
opts += ["--release"]
|
|
elif build_type.is_dev():
|
|
pass # there is no argument for debug
|
|
else:
|
|
opts += ["--profile", build_type.profile]
|
|
|
|
if jobs is not None:
|
|
opts += ["-j", jobs]
|
|
if verbose:
|
|
opts += ["-v"]
|
|
if very_verbose:
|
|
opts += ["-vv"]
|
|
|
|
env = self.build_env()
|
|
self.ensure_bootstrapped()
|
|
self.ensure_clobbered()
|
|
|
|
build_start = time()
|
|
|
|
host = servo.platform.host_triple()
|
|
if 'windows' in host:
|
|
vs_dirs = self.vs_dirs()
|
|
|
|
target_triple = self.cross_compile_target or servo.platform.host_triple()
|
|
if host != target_triple and 'windows' in target_triple:
|
|
if os.environ.get('VisualStudioVersion') or os.environ.get('VCINSTALLDIR'):
|
|
print("Can't cross-compile for Windows inside of a Visual Studio shell.\n"
|
|
"Please run `python mach build [arguments]` to bypass automatic "
|
|
"Visual Studio shell, and make sure the VisualStudioVersion and "
|
|
"VCINSTALLDIR environment variables are not set.")
|
|
sys.exit(1)
|
|
vcinstalldir = vs_dirs['vcdir']
|
|
if not os.path.exists(vcinstalldir):
|
|
print("Can't find Visual C++ %s installation at %s." % (vs_dirs['vs_version'], vcinstalldir))
|
|
sys.exit(1)
|
|
|
|
env['PKG_CONFIG_ALLOW_CROSS'] = "1"
|
|
|
|
if 'windows' in host:
|
|
process = subprocess.Popen('("%s" %s > nul) && "python" -c "import os; print(repr(os.environ))"' %
|
|
(os.path.join(vs_dirs['vcdir'], "Auxiliary", "Build", "vcvarsall.bat"), "x64"),
|
|
stdout=subprocess.PIPE, shell=True)
|
|
stdout, stderr = process.communicate()
|
|
exitcode = process.wait()
|
|
encoding = locale.getpreferredencoding() # See https://stackoverflow.com/a/9228117
|
|
if exitcode == 0:
|
|
decoded = stdout.decode(encoding)
|
|
if decoded.startswith("environ("):
|
|
decoded = decoded.strip()[8:-1]
|
|
os.environ.update(eval(decoded))
|
|
else:
|
|
print("Failed to run vcvarsall. stderr:")
|
|
print(stderr.decode(encoding))
|
|
exit(1)
|
|
|
|
# Gather Cargo build timings (https://doc.rust-lang.org/cargo/reference/timings.html).
|
|
opts = ["--timings"] + opts
|
|
|
|
if very_verbose:
|
|
print(["Calling", "cargo", "build"] + opts)
|
|
for key in env:
|
|
print((key, env[key]))
|
|
|
|
self.download_and_build_android_dependencies_if_needed(env)
|
|
status = self.run_cargo_build_like_command(
|
|
"build", opts, env=env, verbose=verbose,
|
|
libsimpleservo=libsimpleservo, **kwargs
|
|
)
|
|
|
|
# Do some additional things if the build succeeded
|
|
if status == 0:
|
|
if self.is_android_build and not no_package:
|
|
flavor = None
|
|
if "googlevr" in self.features:
|
|
flavor = "googlevr"
|
|
elif "oculusvr" in self.features:
|
|
flavor = "oculusvr"
|
|
rv = Registrar.dispatch("package", context=self.context, build_type=build_type,
|
|
target=self.cross_compile_target, flavor=flavor)
|
|
if rv:
|
|
return rv
|
|
|
|
if sys.platform == "win32":
|
|
servo_exe_dir = os.path.dirname(
|
|
self.get_binary_path(build_type, target=self.cross_compile_target, simpleservo=libsimpleservo)
|
|
)
|
|
assert os.path.exists(servo_exe_dir)
|
|
|
|
build_path = path.join(servo_exe_dir, "build")
|
|
assert os.path.exists(build_path)
|
|
|
|
# on msvc, we need to copy in some DLLs in to the servo.exe dir and the directory for unit tests.
|
|
def package_generated_shared_libraries(libs, build_path, servo_exe_dir):
|
|
for root, dirs, files in os.walk(build_path):
|
|
remaining_libs = list(libs)
|
|
for lib in libs:
|
|
if lib in files:
|
|
shutil.copy(path.join(root, lib), servo_exe_dir)
|
|
remaining_libs.remove(lib)
|
|
continue
|
|
libs = remaining_libs
|
|
if not libs:
|
|
return True
|
|
for lib in libs:
|
|
print("WARNING: could not find " + lib)
|
|
|
|
print("Packaging EGL DLLs")
|
|
egl_libs = ["libEGL.dll", "libGLESv2.dll"]
|
|
if not package_generated_shared_libraries(egl_libs, build_path, servo_exe_dir):
|
|
status = 1
|
|
|
|
# copy needed gstreamer DLLs in to servo.exe dir
|
|
if self.enable_media:
|
|
print("Packaging gstreamer DLLs")
|
|
if not package_gstreamer_dlls(env, servo_exe_dir, target_triple):
|
|
status = 1
|
|
|
|
# UWP app packaging already bundles all required DLLs for us.
|
|
print("Packaging MSVC DLLs")
|
|
if not package_msvc_dlls(servo_exe_dir, target_triple, vs_dirs['vcdir'], vs_dirs['vs_version']):
|
|
status = 1
|
|
|
|
elif sys.platform == "darwin":
|
|
servo_path = self.get_binary_path(
|
|
build_type, target=self.cross_compile_target, simpleservo=libsimpleservo)
|
|
servo_bin_dir = os.path.dirname(servo_path)
|
|
assert os.path.exists(servo_bin_dir)
|
|
|
|
if self.enable_media:
|
|
print("Packaging gstreamer dylibs")
|
|
if not package_gstreamer_dylibs(self.cross_compile_target, servo_path):
|
|
return 1
|
|
|
|
# On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools
|
|
# like Instruments.app.
|
|
try:
|
|
import Cocoa
|
|
icon_path = path.join(self.get_top_dir(), "resources", "servo_1024.png")
|
|
icon = Cocoa.NSImage.alloc().initWithContentsOfFile_(icon_path)
|
|
if icon is not None:
|
|
Cocoa.NSWorkspace.sharedWorkspace().setIcon_forFile_options_(icon,
|
|
servo_path,
|
|
0)
|
|
except ImportError:
|
|
pass
|
|
|
|
# Generate Desktop Notification if elapsed-time > some threshold value
|
|
|
|
elapsed = time() - build_start
|
|
elapsed_delta = datetime.timedelta(seconds=int(elapsed))
|
|
build_message = f"{'Succeeded' if status == 0 else 'Failed'} in {elapsed_delta}"
|
|
self.notify("Servo build", build_message)
|
|
print(build_message)
|
|
|
|
return status
|
|
|
|
def download_and_build_android_dependencies_if_needed(self, env: Dict[str, str]):
|
|
if not self.is_android_build:
|
|
return
|
|
|
|
# Build the name of the package containing all GStreamer dependencies
|
|
# according to the build target.
|
|
android_lib = self.config["android"]["lib"]
|
|
gst_lib = f"gst-build-{android_lib}"
|
|
gst_lib_zip = f"gstreamer-{android_lib}-1.16.0-20190517-095630.zip"
|
|
gst_lib_path = os.path.join(self.target_path, "gstreamer", gst_lib)
|
|
pkg_config_path = os.path.join(gst_lib_path, "pkgconfig")
|
|
env["PKG_CONFIG_PATH"] = pkg_config_path
|
|
if not os.path.exists(gst_lib_path):
|
|
# Download GStreamer dependencies if they have not already been downloaded
|
|
# This bundle is generated with `libgstreamer_android_gen`
|
|
# Follow these instructions to build and deploy new binaries
|
|
# https://github.com/servo/libgstreamer_android_gen#build
|
|
gst_url = f"https://servo-deps-2.s3.amazonaws.com/gstreamer/{gst_lib_zip}"
|
|
print(f"Downloading GStreamer dependencies ({gst_url})")
|
|
|
|
urllib.request.urlretrieve(gst_url, gst_lib_zip)
|
|
zip_ref = zipfile.ZipFile(gst_lib_zip, "r")
|
|
zip_ref.extractall(os.path.join(self.target_path, "gstreamer"))
|
|
os.remove(gst_lib_zip)
|
|
|
|
# Change pkgconfig info to make all GStreamer dependencies point
|
|
# to the libgstreamer_android.so bundle.
|
|
for each in os.listdir(pkg_config_path):
|
|
if each.endswith('.pc'):
|
|
print(f"Setting pkgconfig info for {each}")
|
|
target_path = os.path.join(pkg_config_path, each)
|
|
expr = f"s#libdir=.*#libdir={gst_lib_path}#g"
|
|
subprocess.call(["perl", "-i", "-pe", expr, target_path])
|
|
|
|
@Command('clean',
|
|
description='Clean the target/ and python/_venv[version]/ directories',
|
|
category='build')
|
|
@CommandArgument('--manifest-path',
|
|
default=None,
|
|
help='Path to the manifest to the package to clean')
|
|
@CommandArgument('--verbose', '-v',
|
|
action='store_true',
|
|
help='Print verbose output')
|
|
@CommandArgument('params', nargs='...',
|
|
help="Command-line arguments to be passed through to Cargo")
|
|
def clean(self, manifest_path=None, params=[], verbose=False):
|
|
self.ensure_bootstrapped()
|
|
|
|
virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1])
|
|
virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname)
|
|
if path.exists(virtualenv_path):
|
|
print('Removing virtualenv directory: %s' % virtualenv_path)
|
|
shutil.rmtree(virtualenv_path)
|
|
|
|
opts = ["--manifest-path", manifest_path or path.join(self.context.topdir, "Cargo.toml")]
|
|
if verbose:
|
|
opts += ["-v"]
|
|
opts += params
|
|
return check_call(["cargo", "clean"] + opts, env=self.build_env(), verbose=verbose)
|
|
|
|
def notify(self, title: str, message: str):
|
|
"""Generate desktop notification when build is complete and the
|
|
elapsed build time was longer than 30 seconds.
|
|
|
|
If notify-command is set in the [tools] section of the configuration,
|
|
that is used instead."""
|
|
notify_command = self.config["tools"].get("notify-command")
|
|
|
|
# notifypy does not know how to send transient notifications, so we use a custom
|
|
# notifier on Linux. If transient notifications are not used, then notifications
|
|
# pile up in the notification center and must be cleared manually.
|
|
class LinuxNotifier(notifypy.BaseNotifier):
|
|
def __init__(self, **kwargs):
|
|
pass
|
|
|
|
def send_notification(self, **kwargs):
|
|
try:
|
|
import dbus
|
|
bus = dbus.SessionBus()
|
|
notify_obj = bus.get_object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
|
|
method = notify_obj.get_dbus_method("Notify", "org.freedesktop.Notifications")
|
|
method(
|
|
kwargs.get("application_name"),
|
|
0, # Don't replace previous notification.
|
|
kwargs.get("notification_icon", ""),
|
|
kwargs.get("notification_title"),
|
|
kwargs.get("notification_subtitle"),
|
|
[], # actions
|
|
{"transient": True}, # hints
|
|
-1 # timeout
|
|
)
|
|
except Exception as exception:
|
|
print(f"[Warning] Could not generate notification: {exception}",
|
|
file=sys.stderr)
|
|
return True
|
|
|
|
if notify_command:
|
|
if call([notify_command, title, message]) != 0:
|
|
print("[Warning] Could not generate notification: "
|
|
f"Could not run '{notify_command}'.", file=sys.stderr)
|
|
else:
|
|
notifier = LinuxNotifier if sys.platform.startswith("linux") else None
|
|
notification = notifypy.Notify(use_custom_notifier=notifier)
|
|
notification.title = title
|
|
notification.message = message
|
|
notification.icon = path.join(self.get_top_dir(), "resources", "servo_64.png")
|
|
notification.send(block=False)
|
|
|
|
|
|
def otool(s):
|
|
o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE)
|
|
for line in map(lambda s: s.decode('ascii'), o.stdout):
|
|
if line[0] == '\t':
|
|
yield line.split(' ', 1)[0][1:]
|
|
|
|
|
|
def install_name_tool(binary, *args):
|
|
try:
|
|
subprocess.check_call(['install_name_tool', *args, binary])
|
|
except subprocess.CalledProcessError as e:
|
|
print("install_name_tool exited with return value %d" % e.returncode)
|
|
|
|
|
|
def change_link_name(binary, old, new):
|
|
install_name_tool(binary, '-change', old, f"@executable_path/{new}")
|
|
|
|
|
|
def is_system_library(lib):
|
|
return lib.startswith("/System/Library") or lib.startswith("/usr/lib")
|
|
|
|
|
|
def is_relocatable_library(lib):
|
|
return lib.startswith("@rpath/")
|
|
|
|
|
|
def change_non_system_libraries_path(libraries, relative_path, binary):
|
|
for lib in libraries:
|
|
if is_system_library(lib) or is_relocatable_library(lib):
|
|
continue
|
|
new_path = path.join(relative_path, path.basename(lib))
|
|
change_link_name(binary, lib, new_path)
|
|
|
|
|
|
def resolve_rpath(lib, rpath_root):
|
|
if not is_relocatable_library(lib):
|
|
return lib
|
|
|
|
rpaths = ['', '../', 'gstreamer-1.0/']
|
|
for rpath in rpaths:
|
|
full_path = rpath_root + lib.replace('@rpath/', rpath)
|
|
if path.exists(full_path):
|
|
return path.normpath(full_path)
|
|
|
|
raise Exception("Unable to satisfy rpath dependency: " + lib)
|
|
|
|
|
|
def copy_dependencies(binary_path, lib_path, gst_lib_dir):
|
|
relative_path = path.relpath(lib_path, path.dirname(binary_path)) + "/"
|
|
|
|
# Update binary libraries
|
|
binary_dependencies = set(otool(binary_path))
|
|
change_non_system_libraries_path(binary_dependencies, relative_path, binary_path)
|
|
|
|
plugins = [os.path.join(gst_lib_dir, "gstreamer-1.0", plugin) for plugin in macos_plugins()]
|
|
binary_dependencies = binary_dependencies.union(plugins)
|
|
|
|
# Update dependencies libraries
|
|
need_checked = binary_dependencies
|
|
checked = set()
|
|
while need_checked:
|
|
checking = set(need_checked)
|
|
need_checked = set()
|
|
for f in checking:
|
|
# No need to check these for their dylibs
|
|
if is_system_library(f):
|
|
continue
|
|
full_path = resolve_rpath(f, gst_lib_dir)
|
|
need_relinked = set(otool(full_path))
|
|
new_path = path.join(lib_path, path.basename(full_path))
|
|
if not path.exists(new_path):
|
|
shutil.copyfile(full_path, new_path)
|
|
change_non_system_libraries_path(need_relinked, relative_path, new_path)
|
|
need_checked.update(need_relinked)
|
|
checked.update(checking)
|
|
need_checked.difference_update(checked)
|
|
|
|
|
|
def package_gstreamer_dylibs(cross_compilation_target, servo_bin):
|
|
gst_root = servo.platform.get().gstreamer_root(cross_compilation_target)
|
|
|
|
# This might be None if we are cross-compiling.
|
|
if not gst_root:
|
|
return True
|
|
|
|
lib_dir = path.join(path.dirname(servo_bin), "lib")
|
|
if os.path.exists(lib_dir):
|
|
shutil.rmtree(lib_dir)
|
|
os.mkdir(lib_dir)
|
|
try:
|
|
copy_dependencies(servo_bin, lib_dir, path.join(gst_root, 'lib', ''))
|
|
except Exception as e:
|
|
print("ERROR: could not package required dylibs")
|
|
print(e)
|
|
return False
|
|
return True
|
|
|
|
|
|
def package_gstreamer_dlls(env, servo_exe_dir, target):
|
|
gst_root = servo.platform.get().gstreamer_root(cross_compilation_target=target)
|
|
if not gst_root:
|
|
print("Could not find GStreamer installation directory.")
|
|
return False
|
|
|
|
missing = []
|
|
for gst_lib in windows_dlls():
|
|
try:
|
|
shutil.copy(path.join(gst_root, "bin", gst_lib), servo_exe_dir)
|
|
except Exception:
|
|
missing += [str(gst_lib)]
|
|
|
|
for gst_lib in missing:
|
|
print("ERROR: could not find required GStreamer DLL: " + gst_lib)
|
|
if missing:
|
|
return False
|
|
|
|
# Only copy a subset of the available plugins.
|
|
gst_dlls = windows_plugins()
|
|
|
|
gst_plugin_path_root = os.environ.get("GSTREAMER_PACKAGE_PLUGIN_PATH") or gst_root
|
|
gst_plugin_path = path.join(gst_plugin_path_root, "lib", "gstreamer-1.0")
|
|
if not os.path.exists(gst_plugin_path):
|
|
print("ERROR: couldn't find gstreamer plugins at " + gst_plugin_path)
|
|
return False
|
|
|
|
missing = []
|
|
for gst_lib in gst_dlls:
|
|
try:
|
|
shutil.copy(path.join(gst_plugin_path, gst_lib), servo_exe_dir)
|
|
except Exception:
|
|
missing += [str(gst_lib)]
|
|
|
|
for gst_lib in missing:
|
|
print("ERROR: could not find required GStreamer DLL: " + gst_lib)
|
|
return not missing
|
|
|
|
|
|
def package_msvc_dlls(servo_exe_dir, target, vcinstalldir, vs_version):
|
|
# copy some MSVC DLLs to servo.exe dir
|
|
msvc_redist_dir = None
|
|
vs_platforms = {
|
|
"x86_64": "x64",
|
|
"i686": "x86",
|
|
"aarch64": "arm64",
|
|
}
|
|
target_arch = target.split('-')[0]
|
|
vs_platform = vs_platforms[target_arch]
|
|
vc_dir = vcinstalldir or os.environ.get("VCINSTALLDIR", "")
|
|
if not vs_version:
|
|
vs_version = os.environ.get("VisualStudioVersion", "")
|
|
msvc_deps = [
|
|
"msvcp140.dll",
|
|
"vcruntime140.dll",
|
|
]
|
|
if target_arch != "aarch64" and vs_version in ("14.0", "15.0", "16.0"):
|
|
msvc_deps += ["api-ms-win-crt-runtime-l1-1-0.dll"]
|
|
|
|
# Check if it's Visual C++ Build Tools or Visual Studio 2015
|
|
vs14_vcvars = path.join(vc_dir, "vcvarsall.bat")
|
|
is_vs14 = True if os.path.isfile(vs14_vcvars) or vs_version == "14.0" else False
|
|
if is_vs14:
|
|
msvc_redist_dir = path.join(vc_dir, "redist", vs_platform, "Microsoft.VC140.CRT")
|
|
elif vs_version in ("15.0", "16.0"):
|
|
redist_dir = path.join(vc_dir, "Redist", "MSVC")
|
|
if os.path.isdir(redist_dir):
|
|
for p in os.listdir(redist_dir)[::-1]:
|
|
redist_path = path.join(redist_dir, p)
|
|
for v in ["VC141", "VC142", "VC150", "VC160"]:
|
|
# there are two possible paths
|
|
# `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT`
|
|
redist1 = path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v))
|
|
redist2 = path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v))
|
|
if os.path.isdir(redist1):
|
|
msvc_redist_dir = redist1
|
|
break
|
|
elif os.path.isdir(redist2):
|
|
msvc_redist_dir = redist2
|
|
break
|
|
if msvc_redist_dir:
|
|
break
|
|
if not msvc_redist_dir:
|
|
print("Couldn't locate MSVC redistributable directory")
|
|
return False
|
|
redist_dirs = [
|
|
msvc_redist_dir,
|
|
]
|
|
if "WindowsSdkDir" in os.environ:
|
|
redist_dirs += [path.join(os.environ["WindowsSdkDir"], "Redist", "ucrt", "DLLs", vs_platform)]
|
|
missing = []
|
|
for msvc_dll in msvc_deps:
|
|
for dll_dir in redist_dirs:
|
|
dll = path.join(dll_dir, msvc_dll)
|
|
servo_dir_dll = path.join(servo_exe_dir, msvc_dll)
|
|
if os.path.isfile(dll):
|
|
if os.path.isfile(servo_dir_dll):
|
|
# avoid permission denied error when overwrite dll in servo build directory
|
|
os.chmod(servo_dir_dll, stat.S_IWUSR)
|
|
shutil.copy(dll, servo_exe_dir)
|
|
break
|
|
else:
|
|
missing += [msvc_dll]
|
|
|
|
for msvc_dll in missing:
|
|
print("DLL file `{}` not found!".format(msvc_dll))
|
|
return not missing
|