Auto merge of #21912 - servo:lighter-bootstrap, r=jdm

Add `--build` and `--emulator-x86` to `./mach bootstrap-android`

This allows not downloading dependencies that are not needed for a particular task.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/21912)
<!-- Reviewable:end -->

README.md

@@ -249,7 +249,7 @@ linker = "lld-link.exe"
 
 #### Cross-compilation for Android
 
-Run `./mach bootstrap-android` to get Android-specific tools. See wiki for
+Run `./mach bootstrap-android --build` to get Android-specific tools. See wiki for
 [details](https://github.com/servo/servo/wiki/Building-for-Android).
 
 ## The Rust compiler

etc/ci/bootstrap-android-and-accept-licences.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# We enable pipefail above to satisfy servo-tidy, but disable it again here.
-# In the case of the 'yes' program,
-# exiting when the stdout pipe is broken is expected.
-set +o pipefail
-
-cd $(dirname ${0})/../..
-yes | ./mach bootstrap-android

etc/ci/buildbot_steps.yml

@@ -180,7 +180,7 @@ android:
   - ./mach clean-nightlies --keep 3 --force
   - ./mach clean-cargo-cache --keep 3 --force
   - ./etc/ci/clean_build_artifacts.sh
-  - ./etc/ci/bootstrap-android-and-accept-licences.sh
+  - ./mach bootstrap-android --accept-all-licences --build
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --dev
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --dev
   - bash ./etc/ci/lockfile_changed.sh
@@ -191,7 +191,7 @@ android-mac:
  commands:
   - ./mach clean-nightlies --keep 3 --force
   - ./mach clean-cargo-cache --keep 3 --force
-  - ./etc/ci/bootstrap-android-and-accept-licences.sh
+  - ./mach bootstrap-android --accept-all-licences --build
   - ./mach build --android --dev
   - ./mach package --android --dev
   - bash ./etc/ci/lockfile_changed.sh
@@ -205,7 +205,7 @@ android-x86:
   - ./mach clean-nightlies --keep 3 --force
   - ./mach clean-cargo-cache --keep 3 --force
   - ./etc/ci/clean_build_artifacts.sh
-  - ./etc/ci/bootstrap-android-and-accept-licences.sh
+  - ./mach bootstrap-android --accept-all-licences --build --emulator-x86
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target i686-linux-android --release
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach test-android-startup --release
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach test-wpt-android --release /_mozilla/mozilla/DOMParser.html /_mozilla/mozilla/webgl/context_creation_error.html
@@ -220,7 +220,7 @@ android-nightly:
   - ./mach clean-nightlies --keep 3 --force
   - ./mach clean-cargo-cache --keep 3 --force
   - ./etc/ci/clean_build_artifacts.sh
-  - ./etc/ci/bootstrap-android-and-accept-licences.sh
+  - ./mach bootstrap-android --accept-all-licences --build
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --release
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --release --maven
   - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target=i686-linux-android --release

etc/taskcluster/decision_task.py

@@ -87,7 +87,7 @@ def android_arm32():
         # wget: servo-media-gstreamer’s build script
         .with_script("""
             apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget
-            ./etc/ci/bootstrap-android-and-accept-licences.sh
+            ./mach bootstrap-android --accept-all-licences --build
             ./mach build --android --release
         """)
         .with_artifacts(

python/servo/bootstrap_commands.py

@@ -79,16 +79,24 @@ class MachCommands(CommandBase):
     @Command('bootstrap-android',
              description='Install the Android SDK and NDK.',
              category='bootstrap')
-    def bootstrap_android(self):
+    @CommandArgument('--build',
+                     action='store_true',
+                     help='Install Android-specific dependencies for building')
+    @CommandArgument('--emulator-x86',
+                     action='store_true',
+                     help='Install Android x86 emulator and system image')
+    @CommandArgument('--accept-all-licences',
+                     action='store_true',
+                     help='For non-interactive use')
+    def bootstrap_android(self, build=False, emulator_x86=False, accept_all_licences=False):
+        if not (build or emulator_x86):
+            print("Must specify `--build` or `--emulator-x86` or both.")
 
         ndk = "android-ndk-r12b-{system}-{arch}"
         tools = "sdk-tools-{system}-4333796"
 
-        sdk_build_tools = "27.0.3"
+        emulator_platform = "android-28"
-        emulator_images = [
+        emulator_image = "system-images;%s;google_apis;x86" % emulator_platform
-            ("servo-arm", "25", "google_apis;armeabi-v7a"),
-            ("servo-x86", "28", "google_apis;x86"),
-        ]
 
         known_sha1 = {
             # https://dl.google.com/android/repository/repository2-1.xml
@@ -135,32 +143,34 @@ class MachCommands(CommandBase):
         system = platform.system().lower()
         machine = platform.machine().lower()
         arch = {"i386": "x86"}.get(machine, machine)
-        download("ndk", ndk.format(system=system, arch=arch), flatten=True)
+        if build:
+            download("ndk", ndk.format(system=system, arch=arch), flatten=True)
         download("sdk", tools.format(system=system))
 
-        subprocess.check_call([
+        components = []
-            path.join(toolchains, "sdk", "tools", "bin", "sdkmanager"),
+        if emulator_x86:
-            "platform-tools",
+            components += [
-            "build-tools;" + sdk_build_tools,
+                "platform-tools",
-            "emulator",
+                "emulator",
-        ] + [
+                "platforms;" + emulator_platform,
-            arg
+                emulator_image,
-            for avd_name, api_level, system_image in emulator_images
-            for arg in [
-                "platforms;android-" + api_level,
-                "system-images;android-%s;%s" % (api_level, system_image),
             ]
-        ])
+        if build:
-        for avd_name, api_level, system_image in emulator_images:
+            components += [
-            process = subprocess.Popen(stdin=subprocess.PIPE, stdout=subprocess.PIPE, args=[
+                "platforms;android-18",
-                path.join(toolchains, "sdk", "tools", "bin", "avdmanager"),
+            ]
-                "create", "avd",
+
-                "--path", path.join(toolchains, "avd", avd_name),
+        sdkmanager = [path.join(toolchains, "sdk", "tools", "bin", "sdkmanager")] + components
-                "--name", avd_name,
+        if accept_all_licences:
-                "--package", "system-images;android-%s;%s" % (api_level, system_image),
+            yes = subprocess.Popen(["yes"], stdout=subprocess.PIPE)
-                "--force",
+            process = subprocess.Popen(
-            ])
+                sdkmanager, stdin=yes.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-            output = b""
+            )
+            # Reduce progress bar spam by removing duplicate lines.
+            # Printing the same line again with \r is a no-op in a real terminal,
+            # but each line is shown individually in Taskcluster's log viewer.
+            previous_line = None
+            line = b""
             while 1:
                 # Read one byte at a time because in Python:
                 # * readline() blocks until "\n", which doesn't come before the prompt
@@ -168,6 +178,37 @@ class MachCommands(CommandBase):
                 # * read(n) keeps reading until it gets n bytes or EOF,
                 #   but we don't know reliably how many bytes to read until the prompt
                 byte = process.stdout.read(1)
+                if len(byte) == 0:
+                    print(line)
+                    break
+                line += byte
+                if byte == b'\n' or byte == b'\r':
+                    if line != previous_line:
+                        print(line.decode("utf-8", "replace"), end="")
+                        sys.stdout.flush()
+                    previous_line = line
+                    line = b""
+            exit_code = process.wait()
+            yes.terminate()
+            if exit_code:
+                return exit_code
+        else:
+            subprocess.check_call(sdkmanager)
+
+        if emulator_x86:
+            avd_path = path.join(toolchains, "avd", "servo-x86")
+            process = subprocess.Popen(stdin=subprocess.PIPE, stdout=subprocess.PIPE, args=[
+                path.join(toolchains, "sdk", "tools", "bin", "avdmanager"),
+                "create", "avd",
+                "--path", avd_path,
+                "--name", "servo-x86",
+                "--package", emulator_image,
+                "--force",
+            ])
+            output = b""
+            while 1:
+                # Read one byte at a time, see comment above.
+                byte = process.stdout.read(1)
                 if len(byte) == 0:
                     break
                 output += byte
@@ -175,7 +216,7 @@ class MachCommands(CommandBase):
                 if output.endswith(b"Do you wish to create a custom hardware profile? [no]"):
                     process.stdin.write("no\n")
             assert process.wait() == 0
-            with open(path.join(toolchains, "avd", avd_name, "config.ini"), "a") as f:
+            with open(path.join(avd_path, "config.ini"), "a") as f:
                 f.write("disk.dataPartition.size=2G\n")
 
     @Command('update-hsts-preload',