Auto merge of #21094 - servo:bootstrap-android, r=paul

Add mach bootstrap-android and test-android-startup commands

This adds a `./mach boostrap-android` subcommand that downloads and installs the tools, SDK, NDK,  emulator, and system image for Android. In an environment that can build Servo at all, this should be enough to get all additional dependencies to cross-compile to Android, package an APK, and load it onto a device or an emulator.

At the moment it requires an interactive user to accept the license (and confirm no customization of the emulated virtual device hardware), and then prints environment variables to set for `mach` as well as the command to run to start the emulator (with an already-configured image). A possible next step could be to automate all this, and have `./mach build` run it automatically when needed. (I don’t know if auto-accepting the license is something we should do though.)

This also adds `--emulator` and `--usb` parameters to `./mach install --android` and `./mach run --android`, which tell `adb` what device to pick when both are present. And makes `./mach run --android` print the new process’s PID, for use with e.g. `adb -e logcat --pid 2263`.

Finally, adds the `./mach boostrap-android` subcommand which starts an emulator, installs the APK (it assumes that `mach build` and `mach package` were already executed), runs a single HTML test case, and checks for a message coming from JS through `console.log()` and `adb logcat`.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/21094)
<!-- Reviewable:end -->

.gitignore

@@ -2,6 +2,7 @@
 /.cargo/*
 !/.cargo/config.*
 /.servobuild
+/android-toolchains
 /target
 /ports/android/bin
 /ports/android/libs

components/gfx/platform/freetype/android/font_list.rs

@@ -436,7 +436,6 @@ pub fn for_each_available_family<F>(mut callback: F) where F: FnMut(String) {
 pub fn for_each_variation<F>(family_name: &str, mut callback: F)
     where F: FnMut(String)
 {
-    println!("Variatioooon {:?}", family_name);
     if let Some(family) = FONT_LIST.find_family(family_name) {
         for font in &family.fonts {
             callback(FontList::font_absolute_path(&font.filename));

python/servo/bootstrap_commands.py

@@ -13,6 +13,7 @@ import base64
 import json
 import os
 import os.path as path
+import platform
 import re
 import subprocess
 import sys
@@ -28,7 +29,7 @@ from mach.decorators import (
 
 import servo.bootstrap as bootstrap
 from servo.command_base import CommandBase, cd, check_call
-from servo.util import delete, download_bytes
+from servo.util import delete, download_bytes, download_file, extract, check_hash
 
 
 @CommandProvider
@@ -54,6 +55,111 @@ class MachCommands(CommandBase):
     def bootstrap(self, force=False):
         return bootstrap.bootstrap(self.context, force=force)
 
+    @Command('bootstrap-android',
+             description='Install the Android SDK and NDK.',
+             category='bootstrap')
+    @CommandArgument('--update',
+                     action='store_true',
+                     help='Run SDK component install and emulator image creation again')
+    def bootstrap_android(self, update=False):
+
+        ndk = "android-ndk-r12b-{system}-{arch}"
+        tools = "sdk-tools-{system}-4333796"
+
+        sdk_build_tools = "25.0.2"
+        emulator_images = [
+            ("servo-arm", "25", "google_apis;armeabi-v7a"),
+            ("servo-x86", "28", "google_apis;x86"),
+        ]
+
+        known_sha1 = {
+            # https://dl.google.com/android/repository/repository2-1.xml
+            "sdk-tools-darwin-4333796.zip": "ed85ea7b59bc3483ce0af4c198523ba044e083ad",
+            "sdk-tools-linux-4333796.zip": "8c7c28554a32318461802c1291d76fccfafde054",
+            "sdk-tools-windows-4333796.zip": "aa298b5346ee0d63940d13609fe6bec621384510",
+
+            # https://developer.android.com/ndk/downloads/older_releases
+            "android-ndk-r12b-windows-x86.zip": "8e6eef0091dac2f3c7a1ecbb7070d4fa22212c04",
+            "android-ndk-r12b-windows-x86_64.zip": "337746d8579a1c65e8a69bf9cbdc9849bcacf7f5",
+            "android-ndk-r12b-darwin-x86_64.zip": "e257fe12f8947be9f79c10c3fffe87fb9406118a",
+            "android-ndk-r12b-linux-x86_64.zip": "170a119bfa0f0ce5dc932405eaa3a7cc61b27694",
+        }
+
+        toolchains = path.join(self.context.topdir, "android-toolchains")
+        if not path.isdir(toolchains):
+            os.makedirs(toolchains)
+
+        def download(target_dir, name, flatten=False):
+            final = path.join(toolchains, target_dir)
+            if path.isdir(final):
+                return
+
+            base_url = "https://dl.google.com/android/repository/"
+            filename = name + ".zip"
+            url = base_url + filename
+            archive = path.join(toolchains, filename)
+
+            if not path.isfile(archive):
+                download_file(filename, url, archive)
+            check_hash(archive, known_sha1[filename], "sha1")
+            print("Extracting " + filename)
+            remove = True  # Set to False to avoid repeated downloads while debugging this script
+            if flatten:
+                extracted = final + "_"
+                extract(archive, extracted, remove=remove)
+                contents = os.listdir(extracted)
+                assert len(contents) == 1
+                os.rename(path.join(extracted, contents[0]), final)
+                os.rmdir(extracted)
+            else:
+                extract(archive, final, remove=remove)
+
+        system = platform.system().lower()
+        machine = platform.machine().lower()
+        arch = {"i386": "x86"}.get(machine, machine)
+        download("ndk", ndk.format(system=system, arch=arch), flatten=True)
+        download("sdk", tools.format(system=system))
+
+        subprocess.check_call([
+            path.join(toolchains, "sdk", "tools", "bin", "sdkmanager"),
+            "platform-tools",
+            "build-tools;" + sdk_build_tools,
+            "emulator",
+        ] + [
+            arg
+            for avd_name, api_level, system_image in emulator_images
+            for arg in [
+                "platforms;android-" + api_level,
+                "system-images;android-%s;%s" % (api_level, system_image),
+            ]
+        ])
+        for avd_name, api_level, system_image in emulator_images:
+            process = subprocess.Popen(stdin=subprocess.PIPE, stdout=subprocess.PIPE, args=[
+                path.join(toolchains, "sdk", "tools", "bin", "avdmanager"),
+                "create", "avd",
+                "--path", path.join(toolchains, "avd", avd_name),
+                "--name", avd_name,
+                "--package", "system-images;android-%s;%s" % (api_level, system_image),
+                "--force",
+            ])
+            output = b""
+            while 1:
+                # Read one byte at a time because in Python:
+                # * readline() blocks until "\n", which doesn't come before the prompt
+                # * read() blocks until EOF, which doesn't come before the prompt
+                # * read(n) keeps reading until it gets n bytes or EOF,
+                #   but we don't know reliably how many bytes to read until the prompt
+                byte = process.stdout.read(1)
+                if len(byte) == 0:
+                    break
+                output += byte
+                # There seems to be no way to disable this prompt:
+                if output.endswith(b"Do you wish to create a custom hardware profile? [no]"):
+                    process.stdin.write("no\n")
+            assert process.wait() == 0
+            with open(path.join(toolchains, "avd", avd_name, "config.ini"), "a") as f:
+                f.write("disk.dataPartition.size=2G\n")
+
     @Command('update-hsts-preload',
              description='Download the HSTS preload list',
              category='bootstrap')

python/servo/build_commands.py

@@ -260,10 +260,10 @@ class MachCommands(CommandBase):
             env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -C debug_assertions"
 
         if android:
-            if "ANDROID_NDK" not in os.environ:
+            if "ANDROID_NDK" not in env:
                 print("Please set the ANDROID_NDK environment variable.")
                 sys.exit(1)
-            if "ANDROID_SDK" not in os.environ:
+            if "ANDROID_SDK" not in env:
                 print("Please set the ANDROID_SDK environment variable.")
                 sys.exit(1)
 

python/servo/command_base.py

@@ -525,6 +525,16 @@ class CommandBase(object):
         if self.config["android"]["platform"]:
             env["ANDROID_PLATFORM"] = self.config["android"]["platform"]
 
+        toolchains = path.join(self.context.topdir, "android-toolchains")
+        for kind in ["sdk", "ndk"]:
+            default = os.path.join(toolchains, kind)
+            if os.path.isdir(default):
+                env.setdefault("ANDROID_" + kind.upper(), default)
+
+        tools = os.path.join(toolchains, "sdk", "platform-tools")
+        if os.path.isdir(tools):
+            env["PATH"] = "%s%s%s" % (tools, os.pathsep, env["PATH"])
+
         # These are set because they are the variable names that build-apk
         # expects. However, other submodules have makefiles that reference
         # the env var names above. Once glutin is enabled and set as the
@@ -613,6 +623,13 @@ class CommandBase(object):
                 return sdk_adb
         return "adb"
 
+    def android_emulator_path(self, env):
+        if "ANDROID_SDK" in env:
+            sdk_adb = path.join(env["ANDROID_SDK"], "emulator", "emulator")
+            if path.exists(sdk_adb):
+                return sdk_adb
+        return "emulator"
+
     def handle_android_target(self, target):
         if target == "arm-linux-androideabi":
             self.config["android"]["platform"] = "android-18"

python/servo/package_commands.py

@@ -389,10 +389,16 @@ class PackageCommands(CommandBase):
     @CommandArgument('--android',
                      action='store_true',
                      help='Install on Android')
+    @CommandArgument('--emulator',
+                     action='store_true',
+                     help='For Android, install to the only emulated device')
+    @CommandArgument('--usb',
+                     action='store_true',
+                     help='For Android, install to the only USB device')
     @CommandArgument('--target', '-t',
                      default=None,
                      help='Install the given target platform')
-    def install(self, release=False, dev=False, android=False, target=None):
+    def install(self, release=False, dev=False, android=False, emulator=False, usb=False, target=None):
         env = self.build_env()
         if target and android:
             print("Please specify either --target or --android.")
@@ -416,7 +422,15 @@ class PackageCommands(CommandBase):
 
         if android:
             pkg_path = binary_path + ".apk"
-            exec_command = [self.android_adb_path(env), "install", "-r", pkg_path]
+            exec_command = [self.android_adb_path(env)]
+            if emulator and usb:
+                print("Cannot install to both emulator and USB at the same time.")
+                return 1
+            if emulator:
+                exec_command += ["-e"]
+            if usb:
+                exec_command += ["-d"]
+            exec_command += ["install", "-r", pkg_path]
         elif is_windows():
             pkg_path = path.join(path.dirname(binary_path), 'msi', 'Servo.msi')
             exec_command = ["msiexec", "/i", pkg_path]

python/servo/post_build_commands.py

@@ -45,6 +45,12 @@ class PostBuildCommands(CommandBase):
                      help='Run the dev build')
     @CommandArgument('--android', action='store_true', default=None,
                      help='Run on an Android device through `adb shell`')
+    @CommandArgument('--emulator',
+                     action='store_true',
+                     help='For Android, run in the only emulated device')
+    @CommandArgument('--usb',
+                     action='store_true',
+                     help='For Android, run in the only USB device')
     @CommandArgument('--debug', action='store_true',
                      help='Enable the debugger. Not specifying a '
                           '--debugger option will result in the default '
@@ -64,7 +70,7 @@ class PostBuildCommands(CommandBase):
         'params', nargs='...',
         help="Command-line arguments to be passed through to Servo")
     def run(self, params, release=False, dev=False, android=None, debug=False, debugger=None,
-            headless=False, software=False, bin=None, nightly=None):
+            headless=False, software=False, bin=None, emulator=False, usb=False, nightly=None):
         env = self.build_env()
         env["RUST_BACKTRACE"] = "1"
 
@@ -91,9 +97,19 @@ class PostBuildCommands(CommandBase):
                 ]
             script += [
                 "am start com.mozilla.servo/com.mozilla.servo.MainActivity",
+                "sleep 0.5",
+                "echo Servo PID: $(pidof com.mozilla.servo)",
                 "exit"
             ]
-            shell = subprocess.Popen([self.android_adb_path(env), "shell"], stdin=subprocess.PIPE)
+            args = [self.android_adb_path(env)]
+            if emulator and usb:
+                print("Cannot run in both emulator and USB at the same time.")
+                return 1
+            if emulator:
+                args += ["-e"]
+            if usb:
+                args += ["-d"]
+            shell = subprocess.Popen(args + ["shell"], stdin=subprocess.PIPE)
             shell.communicate("\n".join(script) + "\n")
             return shell.wait()
 
@@ -153,6 +169,18 @@ class PostBuildCommands(CommandBase):
             else:
                 raise e
 
+    @Command('android-emulator',
+             description='Run the Android emulator',
+             category='post-build')
+    @CommandArgument(
+        'args', nargs='...',
+        help="Command-line arguments to be passed through to the emulator")
+    def android_emulator(self, args=None):
+        if not args:
+            print("AVDs created by `./mach bootstrap-android` are servo-arm and servo-x86.")
+        emulator = self.android_emulator_path(self.build_env())
+        return subprocess.call([emulator] + args)
+
     @Command('rr-record',
              description='Run Servo whilst recording execution with rr',
              category='post-build')

python/servo/testing_commands.py

@@ -17,7 +17,7 @@ import os.path as path
 import platform
 import copy
 from collections import OrderedDict
-from time import time
+import time
 import json
 import urllib2
 import urllib
@@ -157,7 +157,7 @@ class MachCommands(CommandBase):
                 print("%s is not a valid test path or suite name" % arg)
                 return 1
 
-        test_start = time()
+        test_start = time.time()
         for suite, tests in selected_suites.iteritems():
             props = suites[suite]
             kwargs = props.get("kwargs", {})
@@ -166,7 +166,7 @@ class MachCommands(CommandBase):
 
             Registrar.dispatch("test-%s" % suite, context=self.context, **kwargs)
 
-        elapsed = time() - test_start
+        elapsed = time.time() - test_start
 
         print("Tests completed in %0.2fs" % elapsed)
 
@@ -551,6 +551,87 @@ class MachCommands(CommandBase):
             output.close()
         return 1
 
+    @Command('test-android-startup',
+             description='Extremely minimal testing of Servo for Android',
+             category='testing')
+    @CommandArgument('--release', '-r', action='store_true',
+                     help='Run the release build')
+    @CommandArgument('--dev', '-d', action='store_true',
+                     help='Run the dev build')
+    def test_android_startup(self, release, dev):
+        if (release and dev) or not (release or dev):
+            print("Please specify one of --dev or --release.")
+            return 1
+        target = "i686-linux-android"
+        print("Assuming --target " + target)
+        env = self.build_env(target=target)
+        assert self.handle_android_target(target)
+
+        emulator_port = "5580"
+        adb = [self.android_adb_path(env), "-s", "emulator-" + emulator_port]
+        emulator_process = subprocess.Popen([
+            self.android_emulator_path(env),
+            "@servo-x86",
+            "-no-window",
+            "-gpu", "guest",
+            "-port", emulator_port,
+        ])
+        try:
+            # This is hopefully enough time for the emulator to exit
+            # if it cannot start because of a configuration problem,
+            # and probably more time than it needs to boot anyway
+            time.sleep(1)
+            if emulator_process.poll() is not None:
+                # The process has terminated already, wait-for-device would block indefinitely
+                return 1
+
+            subprocess.call(adb + ["wait-for-device"])
+
+            # https://stackoverflow.com/a/38896494/1162888
+            while 1:
+                stdout, stderr = subprocess.Popen(
+                    adb + ["shell", "getprop", "sys.boot_completed"],
+                    stdout=subprocess.PIPE,
+                ).communicate()
+                if "1" in stdout:
+                    break
+                print("Waiting for the emulator to boot")
+                time.sleep(1)
+
+            binary_path = self.get_binary_path(release, dev, android=True)
+            result = subprocess.call(adb + ["install", "-r", binary_path + ".apk"])
+            if result != 0:
+                return result
+
+            html = """
+                <script>
+                    console.log("JavaScript is running!")
+                </script>
+            """
+            url = "data:text/html;base64," + html.encode("base64").replace("\n", "")
+            result = subprocess.call(adb + ["shell", """
+                mkdir -p /sdcard/Android/data/com.mozilla.servo/files/
+                echo 'servo' > /sdcard/Android/data/com.mozilla.servo/files/android_params
+                echo '%s' >> /sdcard/Android/data/com.mozilla.servo/files/android_params
+                am start com.mozilla.servo/com.mozilla.servo.MainActivity
+            """ % url])
+            if result != 0:
+                return result
+
+            logcat = adb + ["logcat", "RustAndroidGlueStdouterr:D", "*:S", "-v", "raw"]
+            logcat_process = subprocess.Popen(logcat, stdout=subprocess.PIPE)
+            while 1:
+                line = logcat_process.stdout.readline()
+                if "JavaScript is running!" in line:
+                    print(line)
+                    break
+            logcat_process.kill()
+        finally:
+            try:
+                emulator_process.kill()
+            except OSError:
+                pass
+
     @Command('test-jquery',
              description='Run the jQuery test suite',
              category='testing')

python/servo/util.py

@@ -9,6 +9,7 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import hashlib
 import os
 import os.path
 import platform
@@ -153,9 +154,26 @@ def download_file(desc, src, dst):
     os.rename(tmp_path, dst)
 
 
-def extract(src, dst, movedir=None):
+# https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries
+# In particular, we want the executable bit for executable files.
+class ZipFileWithUnixPermissions(zipfile.ZipFile):
+    def extract(self, member, path=None, pwd=None):
+        if not isinstance(member, zipfile.ZipInfo):
+            member = self.getinfo(member)
+
+        if path is None:
+            path = os.getcwd()
+
+        extracted = self._extract_member(member, path, pwd)
+        mode = os.stat(extracted).st_mode
+        mode |= (member.external_attr >> 16)
+        os.chmod(extracted, mode)
+        return extracted
+
+
+def extract(src, dst, movedir=None, remove=True):
     assert src.endswith(".zip")
-    zipfile.ZipFile(src).extractall(dst)
+    ZipFileWithUnixPermissions(src).extractall(dst)
 
     if movedir:
         for f in os.listdir(movedir):
@@ -164,4 +182,18 @@ def extract(src, dst, movedir=None):
             os.rename(frm, to)
         os.rmdir(movedir)
 
-    os.remove(src)
+    if remove:
+        os.remove(src)
+
+
+def check_hash(filename, expected, algorithm):
+    hasher = hashlib.new(algorithm)
+    with open(filename, "rb") as f:
+        while True:
+            block = f.read(16 * 1024)
+            if len(block) == 0:
+                break
+            hasher.update(block)
+    if hasher.hexdigest() != expected:
+        print("Incorrect {} hash for {}".format(algorithm, filename))
+        sys.exit(1)