Consume official GStreamer binaries on MacOS

This PR re-enables support for the gstreamer mediastack
in macOS by consuming the official binary '.pkg' files
from gstreamer.freedesktop.org

To maintain symmetry with other platforms, the '.pkg'
files are uploaded to servo-build-deps and fetched from
there using the new script 'etc/install_macos_gstreamer.sh'.

Unlike the Homebrew version, the official GStreamer is
distributed as a 'relocatable' framework i.e the dylibs all
have @rpath-relative install names and also link to other
dylibs using @rpath relative path. To address this difference
the 'servo' binary needs to be patched with 'install_name_tool'
to add an LC_RPATH command that sets the relative paths
that the dynamic linker should search when trying to satify
dependencies. In Servo's case, this will be a path relative to
the 'servo' binary itself i.e '@executable_path/lib/'

The additional 'lib' is due to a flaw in the gstreamer
packaging where the install names of some of the dylibs
have the prefix '@rpath/lib' and some of them just have '@rpath'.

This PR also fixes a couple of issues present in the
`mach build` process on MacOS:
1. `mach build` process was not copying transitive dependencies
   of servo binary but only the first level dylibs
2. `mach build` process didn't patch the links to dylibs
   in servo binary (and dependencies). This meant though
   (some) dylibs were copied to local path, the binary
   still loaded the dylibs from system GStreamer installation
   i.e homebrew instead of the copieds dylibs

The build and runtime dependencies in etc/homebrew/Brewfile
and etc/homebrew/Brewfile-build have also been removed in This
PR.

Signed-off-by: Mukilan Thiyagarajan <me@mukilan.in>

.github/workflows/mac.yml

@@ -66,18 +66,11 @@ jobs:
       - name: Bootstrap
         run: |
           python3 -m pip install --upgrade pip virtualenv
-          brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile
-          brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile-build
-          rm -rf /usr/local/etc/openssl
-          rm -rf /usr/local/etc/openssl@1.1
-          brew install openssl@1.1 gnu-tar
+          bash etc/install_macos_gstreamer.sh
+          brew install gnu-tar
       - name: Release build
         run: |
-          export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include"
-          export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib"
-          export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/"
-          export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH"
-          python3 ./mach build --release --media-stack=dummy --with-${{ env.LAYOUT }}
+          python3 ./mach build --release --with-${{ env.LAYOUT }}
       - name: Smoketest
         run: python3 ./mach smoketest
       - name: Unit tests

README.md

@@ -58,8 +58,7 @@ NOTE: run these steps after you've cloned the project locally.
 
 ``` sh
 cd servo 
-brew bundle install --file=etc/homebrew/Brewfile
-brew bundle install --file=etc/homebrew/Brewfile-build
+bash etc/install_macos_gstreamer.sh
 pip install virtualenv
 ```
 

components/servo/lib.rs

@@ -177,6 +177,11 @@ mod media_platform {
         } else {
             let mut plugin_dir = std::env::current_exe().unwrap();
             plugin_dir.pop();
+
+            if cfg!(target_os = "macos") {
+                plugin_dir.push("lib");
+            }
+
             plugin_dir
         };
 

etc/homebrew/Brewfile

@@ -1,10 +0,0 @@
-# Runtime dependencies
-
-brew "gnutls"
-brew "gstreamer"
-brew "gst-plugins-base"
-brew "gst-libav"
-brew "gst-plugins-bad"
-brew "gst-plugins-good"
-brew "gst-rtsp-server"
-brew "openssl"

etc/homebrew/Brewfile-build

@@ -1,12 +0,0 @@
-# Build dependencies (that are not also runtime dependencies)
-
-brew "autoconf@2.13"
-brew "automake"
-brew "cmake"
-brew "pkg-config"
-brew "llvm"
-brew "yasm"
-brew "zlib"
-
-# For sccache
-brew "openssl@1.1"

etc/install_macos_gstreamer.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+VERSION=1.22.2
+URL_BASE=https://github.com/servo/servo-build-deps/releases/download/macOS
+
+cd /tmp
+curl -L "${URL_BASE}/gstreamer-1.0-${VERSION}-universal.pkg" -o gstreamer.pkg
+curl -L "${URL_BASE}/gstreamer-1.0-devel-${VERSION}-universal.pkg" \
+    -o gstreamer-dev.pkg
+sudo installer -pkg 'gstreamer.pkg' -target /
+sudo installer -pkg 'gstreamer-dev.pkg' -target /

python/servo/bootstrap.py

@@ -11,11 +11,29 @@ import distro
 import subprocess
 import six
 import urllib
+
+from os import path
 from subprocess import PIPE
 from zipfile import BadZipfile
 
 import servo.packages as packages
 from servo.util import extract, download_file, host_triple
+from servo.gstreamer import macos_gst_root
+
+
+def check_macos_gstreamer_lib():
+    try:
+        env = os.environ.copy()
+        gst_root = macos_gst_root()
+        env["PATH"] = path.join(gst_root, "bin")
+        env["PKG_CONFIG_PATH"] = path.join(gst_root, "lib", "pkgconfig")
+        has_gst = subprocess.call(["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"],
+                                  stdout=PIPE, stderr=PIPE, env=env) == 0
+        gst_lib_dir = subprocess.check_output(["pkg-config", "--variable=libdir", "gstreamer-1.0"],
+                                              env=env)
+        return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8'))
+    except FileNotFoundError:
+        return False
 
 
 def check_gstreamer_lib():

python/servo/build_commands.py

@@ -34,7 +34,7 @@ from mach.registrar import Registrar
 
 from mach_bootstrap import _get_exec_path
 from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root
-from servo.gstreamer import windows_dlls, windows_plugins, macos_dylibs, macos_plugins
+from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins
 from servo.util import host_triple
 
 
@@ -625,13 +625,20 @@ class MachCommands(CommandBase):
                     status = 1
 
             elif sys.platform == "darwin":
-                servo_exe_dir = os.path.dirname(
-                    self.get_binary_path(release, dev, target=target, simpleservo=libsimpleservo)
-                )
-                assert os.path.exists(servo_exe_dir)
+                servo_path = self.get_binary_path(release, dev, target=target, simpleservo=libsimpleservo)
+                servo_bin_dir = os.path.dirname(servo_path)
+                assert os.path.exists(servo_bin_dir)
 
-                if has_media_stack and not package_gstreamer_dylibs(servo_exe_dir):
-                    return 1
+                if has_media_stack:
+                    gst_root = gstreamer_root(target, env)
+                    if not package_gstreamer_dylibs(gst_root, servo_path):
+                        return 1
+
+                    # On Mac we use the relocatable dylibs from offical gstreamer
+                    # .pkg distribution. We need to add an LC_RPATH to the servo binary
+                    # to allow the dynamic linker to be able to locate these dylibs
+                    # See `man dyld` for more info
+                    add_rpath_to_binary(servo_path, "@executable_path/lib/")
 
                 # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools
                 # like Instruments.app.
@@ -791,22 +798,102 @@ def angle_root(target, nuget_env):
     return angle_default_path
 
 
-def package_gstreamer_dylibs(servo_exe_dir):
-    missing = []
-    gst_dylibs = macos_dylibs() + macos_plugins()
-    for gst_lib in gst_dylibs:
-        try:
-            dest_path = os.path.join(servo_exe_dir, os.path.basename(gst_lib))
-            if os.path.isfile(dest_path):
-                os.remove(dest_path)
-            shutil.copy(gst_lib, servo_exe_dir)
-        except Exception as e:
-            print(e)
-            missing += [str(gst_lib)]
+def otool(s):
+    o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE)
+    for line in map(lambda s: s.decode('ascii'), o.stdout):
+        if line[0] == '\t':
+            yield line.split(' ', 1)[0][1:]
 
-    for gst_lib in missing:
-        print("ERROR: could not find required GStreamer DLL: " + gst_lib)
-    return not missing
+
+def install_name_tool(binary, *args):
+    try:
+        subprocess.check_call(['install_name_tool', *args, binary])
+    except subprocess.CalledProcessError as e:
+        print("install_name_tool exited with return value %d" % e.returncode)
+
+
+def change_link_name(binary, old, new):
+    install_name_tool(binary, '-change', old, f"@executable_path/{new}")
+
+
+def add_rpath_to_binary(binary, relative_path):
+    install_name_tool(binary, "-add_rpath", relative_path)
+
+
+def change_rpath_in_binary(binary, old, new):
+    install_name_tool(binary, "-rpath", old, new)
+
+
+def is_system_library(lib):
+    return lib.startswith("/System/Library") or lib.startswith("/usr/lib")
+
+
+def is_relocatable_library(lib):
+    return lib.startswith("@rpath/")
+
+
+def change_non_system_libraries_path(libraries, relative_path, binary):
+    for lib in libraries:
+        if is_system_library(lib) or is_relocatable_library(lib):
+            continue
+        new_path = path.join(relative_path, path.basename(lib))
+        change_link_name(binary, lib, new_path)
+
+
+def resolve_rpath(lib, rpath_root):
+    if not is_relocatable_library(lib):
+        return lib
+
+    rpaths = ['', '../', 'gstreamer-1.0/']
+    for rpath in rpaths:
+        full_path = rpath_root + lib.replace('@rpath/', rpath)
+        if path.exists(full_path):
+            return path.normpath(full_path)
+
+    raise Exception("Unable to satisfy rpath dependency: " + lib)
+
+
+def copy_dependencies(binary_path, lib_path, gst_root):
+    relative_path = path.relpath(lib_path, path.dirname(binary_path)) + "/"
+
+    # Update binary libraries
+    binary_dependencies = set(otool(binary_path))
+    binary_dependencies = binary_dependencies.union(macos_plugins())
+    change_non_system_libraries_path(binary_dependencies, relative_path, binary_path)
+
+    # Update dependencies libraries
+    need_checked = binary_dependencies
+    checked = set()
+    while need_checked:
+        checking = set(need_checked)
+        need_checked = set()
+        for f in checking:
+            # No need to check these for their dylibs
+            if is_system_library(f):
+                continue
+            full_path = resolve_rpath(f, gst_root)
+            need_relinked = set(otool(full_path))
+            new_path = path.join(lib_path, path.basename(full_path))
+            if not path.exists(new_path):
+                shutil.copyfile(full_path, new_path)
+            change_non_system_libraries_path(need_relinked, relative_path, new_path)
+            need_checked.update(need_relinked)
+        checked.update(checking)
+        need_checked.difference_update(checked)
+
+
+def package_gstreamer_dylibs(gst_root, servo_bin):
+    lib_dir = path.join(path.dirname(servo_bin), "lib")
+    if os.path.exists(lib_dir):
+        shutil.rmtree(lib_dir)
+    os.mkdir(lib_dir)
+    try:
+        copy_dependencies(servo_bin, lib_dir, path.join(gst_root, 'lib', ''))
+    except Exception as e:
+        print("ERROR: could not package required dylibs")
+        print(e)
+        return False
+    return True
 
 
 def package_gstreamer_dlls(env, servo_exe_dir, target, uwp):

python/servo/command_base.py

@@ -36,11 +36,12 @@ import toml
 
 from xml.etree.ElementTree import XML
 from servo.util import download_file
-from .bootstrap import check_gstreamer_lib
+from .bootstrap import check_gstreamer_lib, check_macos_gstreamer_lib
 from mach.decorators import CommandArgument
 from mach.registrar import Registrar
 from servo.packages import WINDOWS_MSVC as msvc_deps
 from servo.util import host_triple
+from servo.gstreamer import macos_gst_root
 
 BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
 NIGHTLY_REPOSITORY_URL = "https://servo-builds2.s3.amazonaws.com/"
@@ -240,6 +241,8 @@ def gstreamer_root(target, env, topdir=None):
             return gst_default_path
     elif is_linux():
         return path.join(topdir, "support", "linux", "gstreamer", "gst")
+    elif is_macosx():
+        return macos_gst_root()
     return None
 
 
@@ -541,6 +544,16 @@ class CommandBase(object):
             return False
         if "media-dummy" in features:
             return False
+
+        if is_macosx():
+            if check_macos_gstreamer_lib():
+                # We override homebrew gstreamer if installed and
+                # always use pkgconfig from official gstreamer framework
+                return True
+            else:
+                raise Exception("Official GStreamer framework not found (we need at least 1.21)."
+                                "Please see installation instructions in README.md")
+
         try:
             if check_gstreamer_lib():
                 return False
@@ -660,14 +673,17 @@ install them, let us know by filing a bug!")
             env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
 
         if is_build and self.needs_gstreamer_env(target or host_triple(), env, uwp, features):
-            gstpath = gstreamer_root(target or host_triple(), env, self.get_top_dir())
-            extra_path += [path.join(gstpath, "bin")]
-            libpath = path.join(gstpath, "lib")
+            gst_root = gstreamer_root(target or host_triple(), env, self.get_top_dir())
+            bin_path = path.join(gst_root, "bin")
+            lib_path = path.join(gst_root, "lib")
+            pkg_config_path = path.join(lib_path, "pkgconfig")
             # we append in the reverse order so that system gstreamer libraries
             # do not get precedence
-            extra_path = [libpath] + extra_path
-            extra_lib = [libpath] + extra_lib
-            append_to_path_env(path.join(libpath, "pkgconfig"), env, "PKG_CONFIG_PATH")
+            extra_path = [bin_path] + extra_path
+            extra_lib = [lib_path] + extra_lib
+            append_to_path_env(pkg_config_path, env, "PKG_CONFIG_PATH")
+            if is_macosx():
+                env["OPENSSL_INCLUDE_DIR"] = path.join(gst_root, "Headers")
 
         if is_linux():
             distrib, version, _ = distro.linux_distribution()

python/servo/gstreamer.py

@@ -9,7 +9,6 @@
 
 import os
 import sys
-import platform
 
 GSTREAMER_DYLIBS = [
     # gstreamer
@@ -121,20 +120,9 @@ def windows_plugins(uwp):
     return [f"{lib}.dll" for lib in libs]
 
 
-def macos_lib_dir():
-    # homebrew use /opt/homebrew on macos ARM, use /usr/local on Intel
-    if platform.machine() == 'arm64':
-        return os.path.join('/', 'opt', 'homebrew', 'lib')
-    return os.path.join('/', 'usr', 'local', 'lib')
-
-
-def macos_dylibs():
-    dylibs = [
-        *[f"lib{lib}-1.0.0.dylib" for lib in GSTREAMER_DYLIBS],
-        "libnice.dylib",
-        "libnice.10.dylib",
-    ]
-    return [os.path.join(macos_lib_dir(), lib) for lib in dylibs]
+def macos_gst_root():
+    return os.path.join(
+        "/", "Library", "Frameworks", "GStreamer.framework", "Versions", "1.0")
 
 
 def macos_plugins():
@@ -148,7 +136,7 @@ def macos_plugins():
     ]
 
     def plugin_path(plugin):
-        return os.path.join(macos_lib_dir(), 'gstreamer-1.0', f"lib{plugin}.dylib")
+        return os.path.join(macos_gst_root(), 'lib', 'gstreamer-1.0', f"lib{plugin}.dylib")
 
     # These plugins depend on the particular version of GStreamer that is installed
     # on the system that is building servo.

python/servo/package_commands.py

@@ -40,7 +40,8 @@ from servo.command_base import (
     is_macosx,
     is_windows,
 )
-from servo.gstreamer import macos_dylibs, macos_plugins
+from servo.build_commands import copy_dependencies, change_rpath_in_binary
+from servo.gstreamer import macos_gst_root
 from servo.util import delete
 
 # Note: mako cannot be imported at the top level because it breaks mach bootstrap
@@ -99,66 +100,11 @@ else:
             raise e
 
 
-def otool(s):
-    o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE)
-    for line in map(lambda s: s.decode('ascii'), o.stdout):
-        if line[0] == '\t':
-            yield line.split(' ', 1)[0][1:]
-
-
 def listfiles(directory):
     return [f for f in os.listdir(directory)
             if path.isfile(path.join(directory, f))]
 
 
-def install_name_tool(old, new, binary):
-    try:
-        subprocess.check_call(['install_name_tool', '-change', old, '@executable_path/' + new, binary])
-    except subprocess.CalledProcessError as e:
-        print("install_name_tool exited with return value %d" % e.returncode)
-
-
-def is_system_library(lib):
-    return lib.startswith("/System/Library") or lib.startswith("/usr/lib")
-
-
-def change_non_system_libraries_path(libraries, relative_path, binary):
-    for lib in libraries:
-        if is_system_library(lib):
-            continue
-        new_path = path.join(relative_path, path.basename(lib))
-        install_name_tool(lib, new_path, binary)
-
-
-def copy_dependencies(binary_path, lib_path):
-    relative_path = path.relpath(lib_path, path.dirname(binary_path)) + "/"
-
-    # Update binary libraries
-    binary_dependencies = set(otool(binary_path))
-    binary_dependencies = binary_dependencies.union(macos_dylibs())
-    binary_dependencies = binary_dependencies.union(macos_plugins())
-    change_non_system_libraries_path(binary_dependencies, relative_path, binary_path)
-
-    # Update dependencies libraries
-    need_checked = binary_dependencies
-    checked = set()
-    while need_checked:
-        checking = set(need_checked)
-        need_checked = set()
-        for f in checking:
-            # No need to check these for their dylibs
-            if is_system_library(f):
-                continue
-            need_relinked = set(otool(f))
-            new_path = path.join(lib_path, path.basename(f))
-            if not path.exists(new_path):
-                shutil.copyfile(f, new_path)
-            change_non_system_libraries_path(need_relinked, relative_path, new_path)
-            need_checked.update(need_relinked)
-        checked.update(checking)
-        need_checked.difference_update(checked)
-
-
 def copy_windows_dependencies(binary_path, destination):
     for f in os.listdir(binary_path):
         if os.path.isfile(path.join(binary_path, f)) and f.endswith(".dll"):
@@ -336,15 +282,16 @@ class PackageCommands(CommandBase):
             shutil.copy2(path.join(dir_to_root, 'Info.plist'), path.join(dir_to_app, 'Contents', 'Info.plist'))
 
             content_dir = path.join(dir_to_app, 'Contents', 'MacOS')
-            os.makedirs(content_dir)
+            lib_dir = path.join(content_dir, 'lib')
+            os.makedirs(lib_dir)
             shutil.copy2(binary_path, content_dir)
 
             change_prefs(dir_to_resources, "macosx")
 
             print("Finding dylibs and relinking")
-            # TODO(mrobinson): GStreamer dependencies don't need to be packaged
-            # with servo until the media backend is re-enabled.
-            # copy_dependencies(path.join(content_dir, 'servo'), content_dir)
+            dmg_binary = path.join(content_dir, "servo")
+            dir_to_gst_lib = path.join(macos_gst_root(), 'lib', '')
+            copy_dependencies(dmg_binary, lib_dir, dir_to_gst_lib)
 
             print("Adding version to Credits.rtf")
             version_command = [binary_path, '--version']
@@ -401,13 +348,17 @@ class PackageCommands(CommandBase):
                 os.remove(tar_path)
             shutil.copytree(path.join(dir_to_root, 'resources'), path.join(dir_to_brew, 'resources'))
             os.makedirs(path.join(dir_to_brew, 'bin'))
-            shutil.copy2(binary_path, path.join(dir_to_brew, 'bin', 'servo'))
             # Note that in the context of Homebrew, libexec is reserved for private use by the formula
             # and therefore is not symlinked into HOMEBREW_PREFIX.
-            os.makedirs(path.join(dir_to_brew, 'libexec'))
-            # TODO(mrobinson): GStreamer dependencies don't need to be packaged
-            # with servo until the media backend is re-enabled.
-            # copy_dependencies(path.join(dir_to_brew, 'bin', 'servo'), path.join(dir_to_brew, 'libexec'))
+            # The 'lib' sub-directory within 'libexec' is necessary to satisfy
+            # rpath relative install_names in the gstreamer packages
+            brew_servo_bin = path.join(dir_to_brew, 'bin', 'servo')
+            shutil.copy2(binary_path, brew_servo_bin)
+            change_rpath_in_binary(
+                brew_servo_bin, '@executable_path/lib/', '@executable_path/../libexec/lib/')
+            dir_to_lib = path.join(dir_to_brew, 'libexec', 'lib')
+            os.makedirs(dir_to_lib)
+            copy_dependencies(brew_servo_bin, dir_to_lib, dir_to_gst_lib)
             archive_deterministically(dir_to_brew, tar_path, prepend_path='servo/')
             delete(dir_to_brew)
             print("Packaged Servo into " + tar_path)