Auto merge of #27302 - jdm:gst-package-mac, r=Manishearth

Make gstreamer packaging explicit

This change aligns Windows and macOS in terms of loading an explicit set of included plugins when initializing gstreamer. It also creates a single source of truth - the set of expected plugins is generated in a build script by the same python code that is used for packaging, so it should be impossible for platforms relying on this system to get out of sync.

Fixes #27293

components/servo/build.rs

@@ -2,6 +2,11 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+use std::env;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+
 fn main() {
     let layout_2013 = std::env::var_os("CARGO_FEATURE_LAYOUT_2013").is_some();
     let layout_2020 = std::env::var_os("CARGO_FEATURE_LAYOUT_2020").is_some();
@@ -12,9 +17,48 @@ fn main() {
     if layout_2013 && layout_2020 {
         error("Must not enable both of the `layout-2013` or `layout-2020` features.")
     }
+
+    println!("cargo:rerun-if-changed=../../python/servo/gstreamer.py");
+
+    let output = Command::new(find_python())
+        .arg("../../python/servo/gstreamer.py")
+        .arg(std::env::var_os("TARGET").unwrap())
+        .output()
+        .unwrap();
+    if !output.status.success() {
+        eprintln!("{}", String::from_utf8_lossy(&output.stdout));
+        eprintln!("{}", String::from_utf8_lossy(&output.stderr));
+        std::process::exit(1)
+    }
+    let path = Path::new(&env::var_os("OUT_DIR").unwrap()).join("gstreamer_plugins.rs");
+    fs::write(path, output.stdout).unwrap();
 }
 
 fn error(message: &str) {
     print!("\n\n    Error: {}\n\n", message);
-    std::process::exit(1)
+    std::process::exit(1);
+}
+
+fn find_python() -> String {
+    env::var("PYTHON2").ok().unwrap_or_else(|| {
+        let candidates = if cfg!(windows) {
+            ["python2.7.exe", "python27.exe", "python.exe"]
+        } else {
+            ["python2.7", "python2", "python"]
+        };
+        for &name in &candidates {
+            if Command::new(name)
+                .arg("--version")
+                .output()
+                .ok()
+                .map_or(false, |out| out.status.success())
+            {
+                return name.to_owned();
+            }
+        }
+        panic!(
+            "Can't find python (tried {})! Try fixing PATH or setting the PYTHON2 env var",
+            candidates.join(", ")
+        )
+    })
 }

components/servo/lib.rs

@@ -134,10 +134,15 @@ pub use servo_url as url;
 
 #[cfg(feature = "media-gstreamer")]
 mod media_platform {
+    #[cfg(any(windows, target_os = "macos"))]
+    mod gstreamer_plugins {
+        include!(concat!(env!("OUT_DIR"), "/gstreamer_plugins.rs"));
+    }
+
     use super::ServoMedia;
     use servo_media_gstreamer::GStreamerBackend;
 
-    #[cfg(target_os = "windows")]
+    #[cfg(feature = "uwp")]
     fn set_gstreamer_log_handler() {
         use gstreamer::{debug_add_log_function, debug_remove_default_log_function, DebugLevel};
 
@@ -165,7 +170,7 @@ mod media_platform {
         });
     }
 
-    #[cfg(windows)]
+    #[cfg(any(windows, target_os = "macos"))]
     pub fn init() {
         // UWP apps have the working directory set appropriately. Win32 apps
         // do not and need some assistance finding the DLLs.
@@ -177,66 +182,24 @@ mod media_platform {
             plugin_dir
         };
 
-        let uwp_plugins = [
+        let backend = match GStreamerBackend::init_with_plugins(
-            "gstapp.dll",
+            plugin_dir,
-            "gstaudioconvert.dll",
+            &gstreamer_plugins::GSTREAMER_PLUGINS,
-            "gstaudiofx.dll",
+        ) {
-            "gstaudioparsers.dll",
-            "gstaudioresample.dll",
-            "gstautodetect.dll",
-            "gstcoreelements.dll",
-            "gstdeinterlace.dll",
-            "gstinterleave.dll",
-            "gstisomp4.dll",
-            "gstlibav.dll",
-            "gstplayback.dll",
-            "gstproxy.dll",
-            "gsttypefindfunctions.dll",
-            "gstvideoconvert.dll",
-            "gstvideofilter.dll",
-            "gstvideoparsersbad.dll",
-            "gstvideoscale.dll",
-            "gstvolume.dll",
-            "gstwasapi.dll",
-        ];
-
-        let non_uwp_plugins = [
-            "gstmatroska.dll",
-            "gstnice.dll",
-            "gstogg.dll",
-            "gstopengl.dll",
-            "gstopus.dll",
-            "gstrtp.dll",
-            "gsttheora.dll",
-            "gstvorbis.dll",
-            "gstvpx.dll",
-            "gstwebrtc.dll",
-        ];
-
-        let plugins: Vec<_> = if cfg!(feature = "uwp") {
-            uwp_plugins.to_vec()
-        } else {
-            uwp_plugins
-                .iter()
-                .map(|&s| s)
-                .chain(non_uwp_plugins.iter().map(|&s| s))
-                .collect()
-        };
-
-        let backend = match GStreamerBackend::init_with_plugins(plugin_dir, &plugins) {
             Ok(b) => b,
             Err(e) => {
-                error!("Error initializing GStreamer: {:?}", e);
+                eprintln!("Error initializing GStreamer: {:?}", e);
-                panic!()
+                std::process::exit(1);
             },
         };
         ServoMedia::init_with_backend(backend);
-        if cfg!(feature = "uwp") {
+        #[cfg(feature = "uwp")]
+        {
             set_gstreamer_log_handler();
         }
     }
 
-    #[cfg(not(windows))]
+    #[cfg(not(any(windows, target_os = "macos")))]
     pub fn init() {
         ServoMedia::init::<GStreamerBackend>();
     }

etc/taskcluster/decision_task.py

@@ -492,6 +492,8 @@ def macos_release_build_with_debug_assertions(priority=None):
             "./etc/ci/lockfile_changed.sh",
             "tar -czf target.tar.gz" +
             " target/release/servo" +
+            " target/release/*.so" +
+            " target/release/*.dylib" +
             " resources",
         ]))
         .with_artifacts("repo/target.tar.gz")

python/servo/build_commands.py

@@ -32,6 +32,7 @@ from mach.registrar import Registrar
 
 from mach_bootstrap import _get_exec_path
 from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root
+from servo.gstreamer import windows_dlls, windows_plugins, macos_dylibs, macos_plugins
 from servo.util import host_triple
 
 
@@ -735,6 +736,14 @@ class MachCommands(CommandBase):
                     status = 1
 
             elif sys.platform == "darwin":
+                servo_exe_dir = os.path.dirname(
+                    self.get_binary_path(release, dev, target=target, simpleservo=libsimpleservo)
+                )
+                assert os.path.exists(servo_exe_dir)
+
+                if not package_gstreamer_dylibs(servo_exe_dir):
+                    return 1
+
                 # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools
                 # like Instruments.app.
                 try:
@@ -842,6 +851,24 @@ def angle_root(target, nuget_env):
     return angle_default_path
 
 
+def package_gstreamer_dylibs(servo_exe_dir):
+    missing = []
+    gst_dylibs = macos_dylibs() + macos_plugins()
+    for gst_lib in gst_dylibs:
+        try:
+            dest_path = os.path.join(servo_exe_dir, os.path.basename(gst_lib))
+            if os.path.isfile(dest_path):
+                os.remove(dest_path)
+            shutil.copy(gst_lib, servo_exe_dir)
+        except Exception as e:
+            print(e)
+            missing += [str(gst_lib)]
+
+    for gst_lib in missing:
+        print("ERROR: could not find required GStreamer DLL: " + gst_lib)
+    return not missing
+
+
 def package_gstreamer_dlls(env, servo_exe_dir, target, uwp):
     gst_root = gstreamer_root(target, env)
     if not gst_root:
@@ -860,29 +887,14 @@ def package_gstreamer_dlls(env, servo_exe_dir, target, uwp):
         "glib-2.0-0.dll",
         "gmodule-2.0-0.dll",
         "gobject-2.0-0.dll",
-        "gstapp-1.0-0.dll",
-        "gstaudio-1.0-0.dll",
-        "gstbase-1.0-0.dll",
-        "gstcodecparsers-1.0-0.dll",
-        "gstcontroller-1.0-0.dll",
-        "gstfft-1.0-0.dll",
-        "gstgl-1.0-0.dll",
-        "gstpbutils-1.0-0.dll",
-        "gstplayer-1.0-0.dll",
-        "gstreamer-1.0-0.dll",
-        "gstriff-1.0-0.dll",
-        "gstrtp-1.0-0.dll",
-        "gstrtsp-1.0-0.dll",
-        "gstsdp-1.0-0.dll",
-        "gsttag-1.0-0.dll",
-        "gstvideo-1.0-0.dll",
-        "gstwebrtc-1.0-0.dll",
         "intl-8.dll",
         "orc-0.4-0.dll",
         "swresample-3.dll",
         "z-1.dll",
     ]
 
+    gst_dlls += windows_dlls(uwp)
+
     if uwp:
         # These come from a more recent version of ffmpeg and
         # aren't present in the official GStreamer 1.16 release.
@@ -897,7 +909,6 @@ def package_gstreamer_dlls(env, servo_exe_dir, target, uwp):
         # with UWP's restrictions.
         gst_dlls += [
             "graphene-1.0-0.dll",
-            "gstsctp-1.0-0.dll",
             "libgmp-10.dll",
             "libgnutls-30.dll",
             "libhogweed-4.dll",
@@ -929,42 +940,7 @@ def package_gstreamer_dlls(env, servo_exe_dir, target, uwp):
         return False
 
     # Only copy a subset of the available plugins.
-    gst_dlls = [
+    gst_dlls = windows_plugins(uwp)
-        "gstapp.dll",
-        "gstaudioconvert.dll",
-        "gstaudiofx.dll",
-        "gstaudioparsers.dll",
-        "gstaudioresample.dll",
-        "gstautodetect.dll",
-        "gstcoreelements.dll",
-        "gstdeinterlace.dll",
-        "gstplayback.dll",
-        "gstinterleave.dll",
-        "gstisomp4.dll",
-        "gstlibav.dll",
-        "gstproxy.dll",
-        "gsttypefindfunctions.dll",
-        "gstvideoconvert.dll",
-        "gstvideofilter.dll",
-        "gstvideoparsersbad.dll",
-        "gstvideoscale.dll",
-        "gstvolume.dll",
-        "gstwasapi.dll",
-    ]
-
-    if not uwp:
-        gst_dlls += [
-            "gstmatroska.dll",
-            "gstnice.dll",
-            "gstogg.dll",
-            "gstopengl.dll",
-            "gstopus.dll",
-            "gstrtp.dll",
-            "gsttheora.dll",
-            "gstvorbis.dll",
-            "gstvpx.dll",
-            "gstwebrtc.dll",
-        ]
 
     gst_plugin_path_root = os.environ.get("GSTREAMER_PACKAGE_PLUGIN_PATH") or gst_root
     gst_plugin_path = path.join(gst_plugin_path_root, "lib", "gstreamer-1.0")

python/servo/gstreamer.py

@@ -0,0 +1,144 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+import sys
+
+GSTREAMER_DYLIBS = [
+    ("gstapp", "gst-plugins-base"),
+    ("gstaudio", "gst-plugins-base"),
+    ("gstbase", "gstreamer"),
+    ("gstcodecparsers", "gst-plugins-bad"),
+    ("gstcontroller", "gstreamer"),
+    ("gstfft", "gst-plugins-base"),
+    ("gstgl", "gst-plugins-base"),
+    ("gstpbutils", "gst-plugins-base"),
+    ("gstplayer", "gst-plugins-bad"),
+    ("gstreamer", "gstreamer"),
+    ("gstriff", "gst-plugins-base"),
+    ("gstrtp", "gst-plugins-base"),
+    ("gstrtsp", "gst-plugins-base"),
+    ("gstsctp", "gst-plugins-bad"),
+    ("gstsdp", "gst-plugins-base"),
+    ("gsttag", "gst-plugins-base"),
+    ("gstvideo", "gst-plugins-base"),
+    ("gstwebrtc", "gst-plugins-bad"),
+]
+
+NON_UWP_DYLIBS = [
+    "gstsctp",
+]
+
+GSTREAMER_PLUGINS = [
+    ("gstapp", "gst-plugins-base"),
+    ("gstaudioconvert", "gst-plugins-base"),
+    ("gstaudiofx", "gst-plugins-good"),
+    ("gstaudioparsers", "gst-plugins-good"),
+    ("gstaudioresample", "gst-plugins-base"),
+    ("gstautodetect", "gst-plugins-good"),
+    ("gstcoreelements", "gstreamer"),
+    ("gstdeinterlace", "gst-plugins-good"),
+    ("gstinterleave", "gst-plugins-good"),
+    ("gstisomp4", "gst-plugins-good"),
+    ("gstlibav", "gst-libav"),
+    ("gstmatroska", "gst-plugins-good"),
+    ("gstogg", "gst-plugins-base"),
+    ("gstopengl", "gst-plugins-base"),
+    ("gstopus", "gst-plugins-base"),
+    ("gstplayback", "gst-plugins-base"),
+    ("gstproxy", "gst-plugins-bad"),
+    ("gstrtp", "gst-plugins-good"),
+    ("gsttheora", "gst-plugins-base"),
+    ("gsttypefindfunctions", "gst-plugins-base"),
+    ("gstvideoconvert", "gst-plugins-base"),
+    ("gstvideofilter", "gst-plugins-good"),
+    ("gstvideoparsersbad", "gst-plugins-bad"),
+    ("gstvideoscale", "gst-plugins-base"),
+    ("gstvorbis", "gst-plugins-base"),
+    ("gstvolume", "gst-plugins-base"),
+    ("gstvpx", "gst-plugins-good"),
+    ("gstwebrtc", "gst-plugins-bad"),
+]
+
+WINDOWS_PLUGINS = [
+    ("gstnice", "gst-plugins-base"),
+    ("gstwasapi", "gst-plugins-base"),
+]
+
+MACOS_PLUGINS = [
+    ("gstapplemedia", "gst-plugins-bad"),
+]
+
+NON_UWP_PLUGINS = [
+    "gstmatroska",
+    "gstnice",
+    "gstogg",
+    "gstopengl",
+    "gstopus",
+    "gstrtp",
+    "gsttheora",
+    "gstvorbis",
+    "gstvpx",
+    "gstwebrtc",
+]
+
+
+def windows_dlls(uwp):
+    dlls = [x for x, _ in GSTREAMER_DYLIBS]
+    if uwp:
+        dlls = filter(lambda x: x not in NON_UWP_DYLIBS, dlls)
+    return [x + "-1.0-0.dll" for x in dlls]
+
+
+def windows_plugins(uwp):
+    dlls = [x for x, _ in GSTREAMER_PLUGINS] + [x for x, _ in WINDOWS_PLUGINS]
+    if uwp:
+        dlls = filter(lambda x: x not in NON_UWP_PLUGINS, dlls)
+    return [x + ".dll" for x in dlls]
+
+
+def macos_dylibs():
+    return [
+        os.path.join(
+            "/usr/local/opt",
+            path,
+            "lib",
+            "lib" + name + "-1.0.0.dylib"
+        ) for name, path in GSTREAMER_DYLIBS
+    ]
+
+
+def macos_plugins():
+    return [
+        os.path.join(
+            "/usr/local/opt",
+            path,
+            "lib",
+            "gstreamer-1.0",
+            "lib" + name + ".so"
+        ) for name, path in GSTREAMER_PLUGINS
+    ]
+
+
+def write_plugin_list(target):
+    plugins = []
+    if "apple-" in target:
+        plugins = [os.path.basename(x) for x in macos_plugins()]
+    elif '-windows-' in target:
+        plugins = windows_plugins('-uwp-' in target)
+    print('''/* This is a generated file. Do not modify. */
+
+pub(crate) static GSTREAMER_PLUGINS: &[&'static str] = &[
+%s
+];
+''' % ',\n'.join(map(lambda x: '"' + x + '"', plugins)))
+
+
+if __name__ == "__main__":
+    write_plugin_list(sys.argv[1])

python/servo/package_commands.py

@@ -39,6 +39,7 @@ from servo.command_base import (
     is_macosx,
     is_windows,
 )
+from servo.gstreamer import macos_dylibs
 from servo.util import delete
 
 # Note: mako cannot be imported at the top level because it breaks mach bootstrap
@@ -139,6 +140,7 @@ def copy_dependencies(binary_path, lib_path):
 
     # Update binary libraries
     binary_dependencies = set(otool(binary_path))
+    binary_dependencies = binary_dependencies.union(macos_dylibs())
     change_non_system_libraries_path(binary_dependencies, relative_path, binary_path)
 
     # Update dependencies libraries