diff --git a/.gitignore b/.gitignore index 65e048a001d..94acd5b1d03 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /.cargo/* !/.cargo/config.* /.servobuild +/android-toolchains /target /ports/android/bin /ports/android/libs diff --git a/components/gfx/platform/freetype/android/font_list.rs b/components/gfx/platform/freetype/android/font_list.rs index e26e6c84e60..3c52e854112 100644 --- a/components/gfx/platform/freetype/android/font_list.rs +++ b/components/gfx/platform/freetype/android/font_list.rs @@ -436,7 +436,6 @@ pub fn for_each_available_family(mut callback: F) where F: FnMut(String) { pub fn for_each_variation(family_name: &str, mut callback: F) where F: FnMut(String) { - println!("Variatioooon {:?}", family_name); if let Some(family) = FONT_LIST.find_family(family_name) { for font in &family.fonts { callback(FontList::font_absolute_path(&font.filename)); diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index b422a553003..699981ff6d1 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -13,6 +13,7 @@ import base64 import json import os import os.path as path +import platform import re import subprocess import sys @@ -28,7 +29,7 @@ from mach.decorators import ( import servo.bootstrap as bootstrap from servo.command_base import CommandBase, cd, check_call -from servo.util import delete, download_bytes +from servo.util import delete, download_bytes, download_file, extract, check_hash @CommandProvider @@ -54,6 +55,111 @@ class MachCommands(CommandBase): def bootstrap(self, force=False): return bootstrap.bootstrap(self.context, force=force) + @Command('bootstrap-android', + description='Install the Android SDK and NDK.', + category='bootstrap') + @CommandArgument('--update', + action='store_true', + help='Run SDK component install and emulator image creation again') + def bootstrap_android(self, update=False): + + ndk = "android-ndk-r12b-{system}-{arch}" + tools = "sdk-tools-{system}-4333796" + + sdk_build_tools = "25.0.2" + emulator_images = [ + ("servo-arm", "25", "google_apis;armeabi-v7a"), + ("servo-x86", "28", "google_apis;x86"), + ] + + known_sha1 = { + # https://dl.google.com/android/repository/repository2-1.xml + "sdk-tools-darwin-4333796.zip": "ed85ea7b59bc3483ce0af4c198523ba044e083ad", + "sdk-tools-linux-4333796.zip": "8c7c28554a32318461802c1291d76fccfafde054", + "sdk-tools-windows-4333796.zip": "aa298b5346ee0d63940d13609fe6bec621384510", + + # https://developer.android.com/ndk/downloads/older_releases + "android-ndk-r12b-windows-x86.zip": "8e6eef0091dac2f3c7a1ecbb7070d4fa22212c04", + "android-ndk-r12b-windows-x86_64.zip": "337746d8579a1c65e8a69bf9cbdc9849bcacf7f5", + "android-ndk-r12b-darwin-x86_64.zip": "e257fe12f8947be9f79c10c3fffe87fb9406118a", + "android-ndk-r12b-linux-x86_64.zip": "170a119bfa0f0ce5dc932405eaa3a7cc61b27694", + } + + toolchains = path.join(self.context.topdir, "android-toolchains") + if not path.isdir(toolchains): + os.makedirs(toolchains) + + def download(target_dir, name, flatten=False): + final = path.join(toolchains, target_dir) + if path.isdir(final): + return + + base_url = "https://dl.google.com/android/repository/" + filename = name + ".zip" + url = base_url + filename + archive = path.join(toolchains, filename) + + if not path.isfile(archive): + download_file(filename, url, archive) + check_hash(archive, known_sha1[filename], "sha1") + print("Extracting " + filename) + remove = True # Set to False to avoid repeated downloads while debugging this script + if flatten: + extracted = final + "_" + extract(archive, extracted, remove=remove) + contents = os.listdir(extracted) + assert len(contents) == 1 + os.rename(path.join(extracted, contents[0]), final) + os.rmdir(extracted) + else: + extract(archive, final, remove=remove) + + system = platform.system().lower() + machine = platform.machine().lower() + arch = {"i386": "x86"}.get(machine, machine) + download("ndk", ndk.format(system=system, arch=arch), flatten=True) + download("sdk", tools.format(system=system)) + + subprocess.check_call([ + path.join(toolchains, "sdk", "tools", "bin", "sdkmanager"), + "platform-tools", + "build-tools;" + sdk_build_tools, + "emulator", + ] + [ + arg + for avd_name, api_level, system_image in emulator_images + for arg in [ + "platforms;android-" + api_level, + "system-images;android-%s;%s" % (api_level, system_image), + ] + ]) + for avd_name, api_level, system_image in emulator_images: + process = subprocess.Popen(stdin=subprocess.PIPE, stdout=subprocess.PIPE, args=[ + path.join(toolchains, "sdk", "tools", "bin", "avdmanager"), + "create", "avd", + "--path", path.join(toolchains, "avd", avd_name), + "--name", avd_name, + "--package", "system-images;android-%s;%s" % (api_level, system_image), + "--force", + ]) + output = b"" + while 1: + # Read one byte at a time because in Python: + # * readline() blocks until "\n", which doesn't come before the prompt + # * read() blocks until EOF, which doesn't come before the prompt + # * read(n) keeps reading until it gets n bytes or EOF, + # but we don't know reliably how many bytes to read until the prompt + byte = process.stdout.read(1) + if len(byte) == 0: + break + output += byte + # There seems to be no way to disable this prompt: + if output.endswith(b"Do you wish to create a custom hardware profile? [no]"): + process.stdin.write("no\n") + assert process.wait() == 0 + with open(path.join(toolchains, "avd", avd_name, "config.ini"), "a") as f: + f.write("disk.dataPartition.size=2G\n") + @Command('update-hsts-preload', description='Download the HSTS preload list', category='bootstrap') diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index d48730252b2..02d71eee8bd 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -260,10 +260,10 @@ class MachCommands(CommandBase): env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -C debug_assertions" if android: - if "ANDROID_NDK" not in os.environ: + if "ANDROID_NDK" not in env: print("Please set the ANDROID_NDK environment variable.") sys.exit(1) - if "ANDROID_SDK" not in os.environ: + if "ANDROID_SDK" not in env: print("Please set the ANDROID_SDK environment variable.") sys.exit(1) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 8e7a9f4b2dc..0963242226b 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -525,6 +525,16 @@ class CommandBase(object): if self.config["android"]["platform"]: env["ANDROID_PLATFORM"] = self.config["android"]["platform"] + toolchains = path.join(self.context.topdir, "android-toolchains") + for kind in ["sdk", "ndk"]: + default = os.path.join(toolchains, kind) + if os.path.isdir(default): + env.setdefault("ANDROID_" + kind.upper(), default) + + tools = os.path.join(toolchains, "sdk", "platform-tools") + if os.path.isdir(tools): + env["PATH"] = "%s%s%s" % (tools, os.pathsep, env["PATH"]) + # These are set because they are the variable names that build-apk # expects. However, other submodules have makefiles that reference # the env var names above. Once glutin is enabled and set as the @@ -613,6 +623,13 @@ class CommandBase(object): return sdk_adb return "adb" + def android_emulator_path(self, env): + if "ANDROID_SDK" in env: + sdk_adb = path.join(env["ANDROID_SDK"], "emulator", "emulator") + if path.exists(sdk_adb): + return sdk_adb + return "emulator" + def handle_android_target(self, target): if target == "arm-linux-androideabi": self.config["android"]["platform"] = "android-18" diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 5f0e3494279..03c2390e308 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -389,10 +389,16 @@ class PackageCommands(CommandBase): @CommandArgument('--android', action='store_true', help='Install on Android') + @CommandArgument('--emulator', + action='store_true', + help='For Android, install to the only emulated device') + @CommandArgument('--usb', + action='store_true', + help='For Android, install to the only USB device') @CommandArgument('--target', '-t', default=None, help='Install the given target platform') - def install(self, release=False, dev=False, android=False, target=None): + def install(self, release=False, dev=False, android=False, emulator=False, usb=False, target=None): env = self.build_env() if target and android: print("Please specify either --target or --android.") @@ -416,7 +422,15 @@ class PackageCommands(CommandBase): if android: pkg_path = binary_path + ".apk" - exec_command = [self.android_adb_path(env), "install", "-r", pkg_path] + exec_command = [self.android_adb_path(env)] + if emulator and usb: + print("Cannot install to both emulator and USB at the same time.") + return 1 + if emulator: + exec_command += ["-e"] + if usb: + exec_command += ["-d"] + exec_command += ["install", "-r", pkg_path] elif is_windows(): pkg_path = path.join(path.dirname(binary_path), 'msi', 'Servo.msi') exec_command = ["msiexec", "/i", pkg_path] diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index 4cca8f80a20..0132d2ea4b0 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -45,6 +45,12 @@ class PostBuildCommands(CommandBase): help='Run the dev build') @CommandArgument('--android', action='store_true', default=None, help='Run on an Android device through `adb shell`') + @CommandArgument('--emulator', + action='store_true', + help='For Android, run in the only emulated device') + @CommandArgument('--usb', + action='store_true', + help='For Android, run in the only USB device') @CommandArgument('--debug', action='store_true', help='Enable the debugger. Not specifying a ' '--debugger option will result in the default ' @@ -64,7 +70,7 @@ class PostBuildCommands(CommandBase): 'params', nargs='...', help="Command-line arguments to be passed through to Servo") def run(self, params, release=False, dev=False, android=None, debug=False, debugger=None, - headless=False, software=False, bin=None, nightly=None): + headless=False, software=False, bin=None, emulator=False, usb=False, nightly=None): env = self.build_env() env["RUST_BACKTRACE"] = "1" @@ -91,9 +97,19 @@ class PostBuildCommands(CommandBase): ] script += [ "am start com.mozilla.servo/com.mozilla.servo.MainActivity", + "sleep 0.5", + "echo Servo PID: $(pidof com.mozilla.servo)", "exit" ] - shell = subprocess.Popen([self.android_adb_path(env), "shell"], stdin=subprocess.PIPE) + args = [self.android_adb_path(env)] + if emulator and usb: + print("Cannot run in both emulator and USB at the same time.") + return 1 + if emulator: + args += ["-e"] + if usb: + args += ["-d"] + shell = subprocess.Popen(args + ["shell"], stdin=subprocess.PIPE) shell.communicate("\n".join(script) + "\n") return shell.wait() @@ -153,6 +169,18 @@ class PostBuildCommands(CommandBase): else: raise e + @Command('android-emulator', + description='Run the Android emulator', + category='post-build') + @CommandArgument( + 'args', nargs='...', + help="Command-line arguments to be passed through to the emulator") + def android_emulator(self, args=None): + if not args: + print("AVDs created by `./mach bootstrap-android` are servo-arm and servo-x86.") + emulator = self.android_emulator_path(self.build_env()) + return subprocess.call([emulator] + args) + @Command('rr-record', description='Run Servo whilst recording execution with rr', category='post-build') diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 545fa19f3b0..f705939ba53 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -17,7 +17,7 @@ import os.path as path import platform import copy from collections import OrderedDict -from time import time +import time import json import urllib2 import urllib @@ -157,7 +157,7 @@ class MachCommands(CommandBase): print("%s is not a valid test path or suite name" % arg) return 1 - test_start = time() + test_start = time.time() for suite, tests in selected_suites.iteritems(): props = suites[suite] kwargs = props.get("kwargs", {}) @@ -166,7 +166,7 @@ class MachCommands(CommandBase): Registrar.dispatch("test-%s" % suite, context=self.context, **kwargs) - elapsed = time() - test_start + elapsed = time.time() - test_start print("Tests completed in %0.2fs" % elapsed) @@ -551,6 +551,87 @@ class MachCommands(CommandBase): output.close() return 1 + @Command('test-android-startup', + description='Extremely minimal testing of Servo for Android', + category='testing') + @CommandArgument('--release', '-r', action='store_true', + help='Run the release build') + @CommandArgument('--dev', '-d', action='store_true', + help='Run the dev build') + def test_android_startup(self, release, dev): + if (release and dev) or not (release or dev): + print("Please specify one of --dev or --release.") + return 1 + target = "i686-linux-android" + print("Assuming --target " + target) + env = self.build_env(target=target) + assert self.handle_android_target(target) + + emulator_port = "5580" + adb = [self.android_adb_path(env), "-s", "emulator-" + emulator_port] + emulator_process = subprocess.Popen([ + self.android_emulator_path(env), + "@servo-x86", + "-no-window", + "-gpu", "guest", + "-port", emulator_port, + ]) + try: + # This is hopefully enough time for the emulator to exit + # if it cannot start because of a configuration problem, + # and probably more time than it needs to boot anyway + time.sleep(1) + if emulator_process.poll() is not None: + # The process has terminated already, wait-for-device would block indefinitely + return 1 + + subprocess.call(adb + ["wait-for-device"]) + + # https://stackoverflow.com/a/38896494/1162888 + while 1: + stdout, stderr = subprocess.Popen( + adb + ["shell", "getprop", "sys.boot_completed"], + stdout=subprocess.PIPE, + ).communicate() + if "1" in stdout: + break + print("Waiting for the emulator to boot") + time.sleep(1) + + binary_path = self.get_binary_path(release, dev, android=True) + result = subprocess.call(adb + ["install", "-r", binary_path + ".apk"]) + if result != 0: + return result + + html = """ + + """ + url = "data:text/html;base64," + html.encode("base64").replace("\n", "") + result = subprocess.call(adb + ["shell", """ + mkdir -p /sdcard/Android/data/com.mozilla.servo/files/ + echo 'servo' > /sdcard/Android/data/com.mozilla.servo/files/android_params + echo '%s' >> /sdcard/Android/data/com.mozilla.servo/files/android_params + am start com.mozilla.servo/com.mozilla.servo.MainActivity + """ % url]) + if result != 0: + return result + + logcat = adb + ["logcat", "RustAndroidGlueStdouterr:D", "*:S", "-v", "raw"] + logcat_process = subprocess.Popen(logcat, stdout=subprocess.PIPE) + while 1: + line = logcat_process.stdout.readline() + if "JavaScript is running!" in line: + print(line) + break + logcat_process.kill() + finally: + try: + emulator_process.kill() + except OSError: + pass + @Command('test-jquery', description='Run the jQuery test suite', category='testing') diff --git a/python/servo/util.py b/python/servo/util.py index 2043b0e2868..d71cffe27da 100644 --- a/python/servo/util.py +++ b/python/servo/util.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals +import hashlib import os import os.path import platform @@ -153,9 +154,26 @@ def download_file(desc, src, dst): os.rename(tmp_path, dst) -def extract(src, dst, movedir=None): +# https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries +# In particular, we want the executable bit for executable files. +class ZipFileWithUnixPermissions(zipfile.ZipFile): + def extract(self, member, path=None, pwd=None): + if not isinstance(member, zipfile.ZipInfo): + member = self.getinfo(member) + + if path is None: + path = os.getcwd() + + extracted = self._extract_member(member, path, pwd) + mode = os.stat(extracted).st_mode + mode |= (member.external_attr >> 16) + os.chmod(extracted, mode) + return extracted + + +def extract(src, dst, movedir=None, remove=True): assert src.endswith(".zip") - zipfile.ZipFile(src).extractall(dst) + ZipFileWithUnixPermissions(src).extractall(dst) if movedir: for f in os.listdir(movedir): @@ -164,4 +182,18 @@ def extract(src, dst, movedir=None): os.rename(frm, to) os.rmdir(movedir) - os.remove(src) + if remove: + os.remove(src) + + +def check_hash(filename, expected, algorithm): + hasher = hashlib.new(algorithm) + with open(filename, "rb") as f: + while True: + block = f.read(16 * 1024) + if len(block) == 0: + break + hasher.update(block) + if hasher.hexdigest() != expected: + print("Incorrect {} hash for {}".format(algorithm, filename)) + sys.exit(1)