Auto merge of #21586 - Manishearth:gstreamer-bootstrap, r=jdm

Make bootstrap handle linux setup, include gstreamer, make bootstrap run standalone

Previously `./mach bootstrap` wasn't very useful on Linux, it would just install salt. This PR moves that to `./mach bootstrap-salt`, and has `./mach bootstrap` perform the installation steps mentioned in the README.

With these changes `./mach bootstrap` is a valid way of setting up the servo build. Additionally, this adds gstreamer setup to this for cases where gstreamer is not up to date.

Edit: And now, mach bootstrap is able to be run on a fresh system with just python installed (no need for virtualenv or libffi).

Fixes https://github.com/servo/servo/issues/21446 , fixes https://github.com/servo/servo/issues/21417

r? @jdm

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/21586)
<!-- Reviewable:end -->

README.md

@@ -78,6 +78,10 @@ If you've already partially compiled servo but forgot to do this step, run `./ma
 
 #### On Debian-based Linuxes
 
+Please run `./mach bootstrap`.
+
+If this doesn't work, file a bug, and, run the commands below:
+
 ``` sh
 sudo apt install git curl autoconf libx11-dev \
     libfreetype6-dev libgl1-mesa-dri libglib2.0-dev xorg-dev \
@@ -89,15 +93,50 @@ sudo apt install git curl autoconf libx11-dev \
 ```
 
 If you using a version prior to **Ubuntu 17.04** or **Debian Sid**, replace `libssl1.0-dev` with `libssl-dev`.
+Additionally, you'll need a local copy of GStreamer with a version later than 12.0. You can place it in `support/linux/gstreamer/gstreamer`, or run `./mach bootstrap-gstreamer` to set it up.
 
 If you are using **Ubuntu 16.04** run `export HARFBUZZ_SYS_NO_PKG_CONFIG=1` before building to avoid an error with harfbuzz.
 
-If you are on **Ubuntu 14.04** and encountered errors on installing these dependencies involving `libcheese`, see [#6158](https://github.com/servo/servo/issues/6158) for a workaround.
+If you are on **Ubuntu 14.04** and encountered errors on installing these dependencies involving `libcheese`, see [#6158](https://github.com/servo/servo/issues/6158) for a workaround. You may also need to install gcc 4.9, clang 4.0, and cmake 3.2:
+
+<details>
+gcc 4.9:
+
+```sh
+sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+sudo apt-get update
+sudo apt-get install gcc-4.9 g++-4.9
+sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-4.9
+```
+
+clang 4.0:
+
+```sh
+wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+sudo apt-add-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-4.0 main"
+sudo apt-get update
+sudo apt-get install -y clang-4.0
+```
+
+cmake 3.2:
+
+```sh
+sudo apt-get install software-properties-common
+sudo add-apt-repository ppa:george-edison55/cmake-3.x
+sudo apt-get update
+sudo apt-get install cmake
+```
+
+</details>
 
 If `virtualenv` does not exist, try `python-virtualenv`.
 
 #### On Fedora
 
+Please run `./mach bootstrap`.
+
+If this doesn't work, file a bug, and, run the commands below:
+
 ``` sh
 sudo dnf install curl libtool gcc-c++ libXi-devel \
     freetype-devel mesa-libGL-devel mesa-libEGL-devel glib2-devel libX11-devel libXrandr-devel gperf \
@@ -108,6 +147,11 @@ sudo dnf install curl libtool gcc-c++ libXi-devel \
 ```
 #### On CentOS
 
+
+Please run `./mach bootstrap`.
+
+If this doesn't work, file a bug, and, run the commands below:
+
 ``` sh
 sudo yum install curl libtool gcc-c++ libXi-devel \
     freetype-devel mesa-libGL-devel mesa-libEGL-devel glib2-devel libX11-devel libXrandr-devel gperf \

mach

@@ -20,8 +20,11 @@ def main(args):
     topdir = os.path.abspath(os.path.dirname(sys.argv[0]))
     sys.path.insert(0, os.path.join(topdir, "python"))
     import mach_bootstrap
-    mach = mach_bootstrap.bootstrap(topdir)
-    sys.exit(mach.run(sys.argv[1:]))
+    if len(sys.argv) > 1 and sys.argv[1] == "bootstrap":
+        sys.exit(mach_bootstrap.bootstrap_command_only(topdir))
+    else:
+        mach = mach_bootstrap.bootstrap(topdir)
+        sys.exit(mach.run(sys.argv[1:]))
 
 
 if __name__ == '__main__':

python/mach_bootstrap.py

@@ -224,11 +224,29 @@ def _is_windows():
     return sys.platform == 'win32'
 
 
+class DummyContext(object):
+    pass
+
+
+def bootstrap_command_only(topdir):
+    from servo.bootstrap import bootstrap
+
+    context = DummyContext()
+    context.topdir = topdir
+    force = False
+    if len(sys.argv) == 3 and sys.argv[2] == "-f":
+        force = True
+    bootstrap(context, force)
+    return 0
+
+
 def bootstrap(topdir):
     _ensure_case_insensitive_if_windows()
 
     topdir = os.path.abspath(topdir)
 
+    len(sys.argv) > 1 and sys.argv[1] == "bootstrap"
+
     # We don't support paths with Unicode characters for now
     # https://github.com/servo/servo/issues/10002
     try:

python/servo/bootstrap.py

@@ -17,22 +17,57 @@ import servo.packages as packages
 from servo.util import extract, download_file, host_triple
 
 
-def run_as_root(command):
+def install_trusty_deps(force):
+    version = str(subprocess.check_output(['gcc', '-dumpversion'])).split('.')
+    gcc = True
+    if int(version[0]) > 4:
+        gcc = False
+    elif int(version[0]) == 4 and int(version[1]) >= 9:
+        gcc = False
+
+    version = str(subprocess.check_output(['clang', '-dumpversion'])).split('.')
+    clang = int(version[0]) < 4
+
+    if gcc:
+        run_as_root(["add-apt-repository", "ppa:ubuntu-toolchain-r/test"], force)
+        run_as_root(["apt-get", "update"])
+        run_as_root(["apt-get", "install", "gcc-4.9", "g++-4.9"], force)
+        run_as_root(['update-alternatives', '--install', '/usr/bin/gcc', 'gcc',
+                     '/usr/bin/gcc-4.9', '60', '--slave', '/usr/bin/g++', 'g++',
+                     '/usr/bin/g++-4.9'])
+    if clang:
+        run_as_root(["bash", "-c", 'wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -'])
+        run_as_root(["apt-add-repository", "deb http://apt.llvm.org/trusty/ llvm-toolchain-xenial-4.0 main"], force)
+        run_as_root(["apt-get", "update"])
+        run_as_root(["apt-get", "install", "clang-4.0"], force)
+
+    return gcc or clang
+
+
+def check_gstreamer_lib():
+    return subprocess.call(["pkg-config", "gstreamer-1.0 >= 1.12"],
+                           stdout=PIPE, stderr=PIPE) == 0
+
+
+def run_as_root(command, force=False):
     if os.geteuid() != 0:
         command.insert(0, 'sudo')
+    if force:
+        command += "-y"
     return subprocess.call(command)
 
 
-def install_salt_dependencies(context, force):
+def install_linux_deps(context, pkgs_ubuntu, pkgs_fedora, force):
     install = False
+    pkgs = []
     if context.distro == 'Ubuntu':
-        pkgs = ['build-essential', 'libssl-dev', 'libffi-dev', 'python-dev']
         command = ['apt-get', 'install']
+        pkgs = pkgs_ubuntu
         if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0:
             install = True
     elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora']:
         installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
-        pkgs = ['gcc', 'libffi-devel', 'python-devel', 'openssl-devel']
+        pkgs = pkgs_fedora
         for p in pkgs:
             command = ['dnf', 'install']
             if "|{}".format(p) not in installed_pkgs:
@@ -42,8 +77,74 @@ def install_salt_dependencies(context, force):
     if install:
         if force:
             command.append('-y')
-        print("Installing missing Salt dependencies...")
+        print("Installing missing dependencies...")
         run_as_root(command + pkgs)
+        return True
+    return False
+
+
+def install_salt_dependencies(context, force):
+    pkgs_apt = ['build-essential', 'libssl-dev', 'libffi-dev', 'python-dev']
+    pkgs_dnf = ['gcc', 'libffi-devel', 'python-devel', 'openssl-devel']
+    if not install_linux_deps(context, pkgs_apt, pkgs_dnf, force):
+        print("Dependencies are already installed")
+
+
+def gstreamer(context, force=False):
+    cur = os.curdir
+    gstdir = os.path.join(cur, "support", "linux", "gstreamer")
+    if not os.path.isdir(os.path.join(gstdir, "gstreamer", "lib")):
+        subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
+        return True
+    return False
+
+
+def linux(context, force=False):
+    # Please keep these in sync with the packages in README.md
+    pkgs_apt = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev',
+                'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++',
+                'build-essential', 'cmake', 'python-pip',
+                'libbz2-dev', 'libosmesa6-dev', 'libxmu6', 'libxmu-dev', 'libglu1-mesa-dev',
+                'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev', 'libharfbuzz-dev',
+                'ccache', 'clang', 'autoconf2.13']
+    pkgs_dnf = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel',
+                'mesa-libGL-devel', 'mesa-libEGL-devel', 'glib2-devel', 'libX11-devel',
+                'libXrandr-devel', 'gperf', 'fontconfig-devel', 'cabextract', 'ttmkfdir',
+                'python2', 'python2-virtualenv', 'python2-pip', 'expat-devel', 'rpm-build',
+                'openssl-devel', 'cmake', 'bzip2-devel', 'libXcursor-devel', 'libXmu-devel',
+                'mesa-libOSMesa-devel', 'dbus-devel', 'ncurses-devel', 'harfbuzz-devel',
+                'ccache', 'mesa-libGLU-devel', 'clang', 'clang-libs', 'gstreamer1-devel',
+                'gstreamer1-plugins-base-devel', 'gstreamer1-plugins-bad-free-devel', 'autoconf213']
+    if context.distro == "Ubuntu":
+        if context.distro_version == "17.04":
+            pkgs_apt += ["libssl-dev"]
+        elif int(context.distro_version.split(".")[0]) < 17:
+            pkgs_apt += ["libssl-dev"]
+        else:
+            pkgs_apt += ["libssl1.0-dev"]
+
+        if context.distro_version == "14.04":
+            pkgs_apt += ["python-virtualenv"]
+        else:
+            pkgs_apt += ["virtualenv"]
+            pkgs_apt += ['libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev',
+                         'libgstreamer-plugins-bad1.0-dev']
+
+    elif context.distro == "Debian" and context.distro_version == "Sid":
+        pkgs_apt += ["libssl-dev"]
+    else:
+        pkgs_apt += ["libssl1.0-dev"]
+
+    installed_something = install_linux_deps(context, pkgs_apt, pkgs_dnf, force)
+
+    if not check_gstreamer_lib():
+        installed_something |= gstreamer(context, force)
+
+    if context.distro == "Ubuntu" and context.distro_version == "14.04":
+        installed_something |= install_trusty_deps(force)
+
+    if not installed_something:
+        print("Dependencies were already installed!")
 
 
 def salt(context, force=False):
@@ -226,11 +327,16 @@ def windows_msvc(context, force=False):
     return 0
 
 
-def bootstrap(context, force=False):
+LINUX_SPECIFIC_BOOTSTRAPPERS = {
+    "salt": salt,
+    "gstreamer": gstreamer,
+}
+
+
+def bootstrap(context, force=False, specific=None):
     '''Dispatches to the right bootstrapping function for the OS.'''
 
     bootstrapper = None
-
     if "windows-msvc" in host_triple():
         bootstrapper = windows_msvc
     elif "linux-gnu" in host_triple():
@@ -243,7 +349,10 @@ def bootstrap(context, force=False):
             'ubuntu',
         ]:
             context.distro = distro
-            bootstrapper = salt
+            context.distro_version = version
+            bootstrapper = LINUX_SPECIFIC_BOOTSTRAPPERS.get(specific, linux)
+        else:
+            raise Exception("mach bootstrap does not support %s, please file a bug" % distro)
 
     if bootstrapper is None:
         print('Bootstrap support is not yet available for your OS.')

python/servo/bootstrap_commands.py

@@ -53,8 +53,29 @@ class MachCommands(CommandBase):
                      action='store_true',
                      help='Boostrap without confirmation')
     def bootstrap(self, force=False):
+        # This entry point isn't actually invoked, ./mach bootstrap is directly
+        # called by mach (see mach_bootstrap.bootstrap_command_only) so that
+        # it can install dependencies without needing mach's dependencies
         return bootstrap.bootstrap(self.context, force=force)
 
+    @Command('bootstrap-salt',
+             description='Install and set up the salt environment.',
+             category='bootstrap')
+    @CommandArgument('--force', '-f',
+                     action='store_true',
+                     help='Boostrap without confirmation')
+    def bootstrap_salt(self, force=False):
+        return bootstrap.bootstrap(self.context, force=force, specific="salt")
+
+    @Command('bootstrap-gstreamer',
+             description='Set up a local copy of the gstreamer libraries (linux only).',
+             category='bootstrap')
+    @CommandArgument('--force', '-f',
+                     action='store_true',
+                     help='Boostrap without confirmation')
+    def bootstrap_gstreamer(self, force=False):
+        return bootstrap.bootstrap(self.context, force=force, specific="gstreamer")
+
     @Command('bootstrap-android',
              description='Install the Android SDK and NDK.',
              category='bootstrap')

python/servo/command_base.py

@@ -24,6 +24,7 @@ import tarfile
 from xml.etree.ElementTree import XML
 from servo.util import download_file
 import urllib2
+from bootstrap import check_gstreamer_lib
 
 from mach.registrar import Registrar
 import toml
@@ -347,6 +348,9 @@ class CommandBase(object):
         build_type = "release" if release else "debug"
         return path.join(base_path, build_type, apk_name)
 
+    def get_gstreamer_path(self):
+        return path.join(self.context.topdir, "support", "linux", "gstreamer", "gstreamer")
+
     def get_binary_path(self, release, dev, android=False):
         # TODO(autrilla): this function could still use work - it shouldn't
         # handle quitting, or printing. It should return the path, or an error.
@@ -476,6 +480,37 @@ class CommandBase(object):
             bin_folder = path.join(destination_folder, "PFiles", "Mozilla research", "Servo Tech Demo")
         return path.join(bin_folder, "servo{}".format(BIN_SUFFIX))
 
+    def needs_gstreamer_env(self, target):
+        try:
+            if check_gstreamer_lib():
+                return False
+        except:
+            # Some systems don't have pkg-config; we can't probe in this case
+            # and must hope for the best
+            return False
+        effective_target = target or host_triple()
+        if "x86_64" not in effective_target or "android" in effective_target:
+            # We don't build gstreamer for non-x86_64 / android yet
+            return False
+        if sys.platform == "linux2":
+            if path.isdir(self.get_gstreamer_path()):
+                return True
+            else:
+                raise Exception("Your system's gstreamer libraries are out of date \
+(we need at least 1.12). Please run ./mach bootstrap-gstreamer")
+        else:
+                raise Exception("Your system's gstreamer libraries are out of date \
+(we need at least 1.12). If you're unable to \
+install them, let us know by filing a bug!")
+        return False
+
+    def set_run_env(self, android=False):
+        """Some commands, like test-wpt, don't use a full build env,
+           but may still need dynamic search paths. This command sets that up"""
+        if not android and self.needs_gstreamer_env(None):
+            gstpath = self.get_gstreamer_path()
+            os.environ["LD_LIBRARY_PATH"] = path.join(gstpath, "lib", "x86_64-linux-gnu")
+
     def build_env(self, hosts_file_path=None, target=None, is_build=False, test_unit=False):
         """Return an extended environment dictionary."""
         env = os.environ.copy()
@@ -513,12 +548,26 @@ class CommandBase(object):
             # Link LLVM
             env["LIBCLANG_PATH"] = path.join(package_dir("llvm"), "lib")
 
-        if is_windows():
             if not os.environ.get("NATIVE_WIN32_PYTHON"):
                 env["NATIVE_WIN32_PYTHON"] = sys.executable
             # Always build harfbuzz from source
             env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
 
+        if self.needs_gstreamer_env(target):
+            gstpath = self.get_gstreamer_path()
+            extra_path += [path.join(gstpath, "bin")]
+            libpath = path.join(gstpath, "lib", "x86_64-linux-gnu")
+            # we append in the reverse order so that system gstreamer libraries
+            # do not get precedence
+            extra_path = [libpath] + extra_path
+            extra_lib = [libpath] + extra_path
+            append_to_path_env(path.join(libpath, "pkgconfig"), env, "PKG_CONFIG_PATH")
+
+        if sys.platform == "linux2":
+            distro, version, _ = platform.linux_distribution()
+            if distro == "Ubuntu" and (version == "16.04" or version == "14.04"):
+                env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
+
         if extra_path:
             env["PATH"] = "%s%s%s" % (os.pathsep.join(extra_path), os.pathsep, env["PATH"])
 

python/servo/package_commands.py

@@ -184,7 +184,6 @@ class PackageCommands(CommandBase):
                      default=None,
                      help='Package using the given Gradle flavor')
     def package(self, release=False, dev=False, android=None, debug=False, debugger=None, target=None, flavor=None):
-        env = self.build_env()
         if android is None:
             android = self.config["build"]["android"]
         if target and android:
@@ -192,6 +191,9 @@ class PackageCommands(CommandBase):
             sys.exit(1)
         if not android:
             android = self.handle_android_target(target)
+        else:
+            target = self.config["android"]["target"]
+        env = self.build_env(target=target)
         binary_path = self.get_binary_path(release, dev, android=android)
         dir_to_root = self.get_top_dir()
         target_dir = path.dirname(binary_path)

python/servo/testing_commands.py

@@ -380,9 +380,10 @@ class MachCommands(CommandBase):
             binary_args=self.in_android_emulator(release, dev) + (binary_args or []),
             binary=sys.executable,
         )
-        return self._test_wpt(**kwargs)
+        return self._test_wpt(android=True, **kwargs)
 
-    def _test_wpt(self, **kwargs):
+    def _test_wpt(self, android=False, **kwargs):
+        self.set_run_env(android)
         hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts')
         os.environ["hosts_file_path"] = hosts_file_path
         run_file = path.abspath(path.join(self.context.topdir, "tests", "wpt", "run.py"))

python/servo/util.py

@@ -20,7 +20,6 @@ import StringIO
 import sys
 import zipfile
 import urllib2
-import certifi
 
 
 try:
@@ -30,6 +29,7 @@ except ImportError:
 
 # The cafile parameter was added in 2.7.9
 if HAS_SNI and sys.version_info >= (2, 7, 9):
+    import certifi
     STATIC_RUST_LANG_ORG_DIST = "https://static.rust-lang.org/dist"
     URLOPEN_KWARGS = {"cafile": certifi.where()}
 else:

servo-tidy.toml

@@ -61,6 +61,7 @@ files = [
   "./resources/hsts_preload.json",
   "./tests/wpt/metadata/MANIFEST.json",
   "./support/android/openssl.sh",
+  "./support/linux/gstreamer/gstreamer.sh",
   # Upstream code from Khronos/WebGL uses tabs for indentation
   "./tests/wpt/webgl/tests",
   # Our import script is not currently respecting the lint.

support/linux/gstreamer/.gitignore

@@ -0,0 +1,2 @@
+gstreamer/
+

support/linux/gstreamer/gstreamer.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set -o errexit
+
+wget http://servo-deps.s3.amazonaws.com/gstreamer/gstreamer-x86_64-linux-gnu.tar.gz -O gstreamer.tar.gz
+tar -zxf gstreamer.tar.gz
+rm gstreamer.tar.gz
+sed -i "s;prefix=/root/gstreamer;prefix=${PWD}/gstreamer;g" ${PWD}/gstreamer/lib/x86_64-linux-gnu/pkgconfig/*.pc