From 89e1878bd6dcc7f7304abbf385ca4a76356b6313 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 11 Oct 2018 15:22:22 +0200 Subject: [PATCH 1/3] Add `--build` and `--emulator-x86` to `./mach bootstrap-android` This allows not downloading dependencies that are not needed for a praticular task. --- README.md | 2 +- .../bootstrap-android-and-accept-licences.sh | 2 +- etc/ci/buildbot_steps.yml | 8 +-- etc/taskcluster/decision_task.py | 2 +- python/servo/bootstrap_commands.py | 58 +++++++++++-------- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 9f09fb8a1a1..4f14dda1910 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ linker = "lld-link.exe" #### Cross-compilation for Android -Run `./mach bootstrap-android` to get Android-specific tools. See wiki for +Run `./mach bootstrap-android --build` to get Android-specific tools. See wiki for [details](https://github.com/servo/servo/wiki/Building-for-Android). ## The Rust compiler diff --git a/etc/ci/bootstrap-android-and-accept-licences.sh b/etc/ci/bootstrap-android-and-accept-licences.sh index dde62438efa..1ad22868184 100755 --- a/etc/ci/bootstrap-android-and-accept-licences.sh +++ b/etc/ci/bootstrap-android-and-accept-licences.sh @@ -14,4 +14,4 @@ set -o pipefail set +o pipefail cd $(dirname ${0})/../.. -yes | ./mach bootstrap-android +yes | ./mach bootstrap-android "${@}" diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index da27cb4db1f..0ceaffc7f37 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -180,7 +180,7 @@ android: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - ./etc/ci/clean_build_artifacts.sh - - ./etc/ci/bootstrap-android-and-accept-licences.sh + - ./etc/ci/bootstrap-android-and-accept-licences.sh --build - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --dev - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --dev - bash ./etc/ci/lockfile_changed.sh @@ -191,7 +191,7 @@ android-mac: commands: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - - ./etc/ci/bootstrap-android-and-accept-licences.sh + - ./etc/ci/bootstrap-android-and-accept-licences.sh --build - ./mach build --android --dev - ./mach package --android --dev - bash ./etc/ci/lockfile_changed.sh @@ -205,7 +205,7 @@ android-x86: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - ./etc/ci/clean_build_artifacts.sh - - ./etc/ci/bootstrap-android-and-accept-licences.sh + - ./etc/ci/bootstrap-android-and-accept-licences.sh --build --emulator-x86 - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target i686-linux-android --release - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach test-android-startup --release - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach test-wpt-android --release /_mozilla/mozilla/DOMParser.html /_mozilla/mozilla/webgl/context_creation_error.html @@ -220,7 +220,7 @@ android-nightly: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - ./etc/ci/clean_build_artifacts.sh - - ./etc/ci/bootstrap-android-and-accept-licences.sh + - ./etc/ci/bootstrap-android-and-accept-licences.sh --build - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --release - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --release --maven - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target=i686-linux-android --release diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 24aa16ac629..b81777aea63 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -87,7 +87,7 @@ def android_arm32(): # wget: servo-media-gstreamer’s build script .with_script(""" apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget - ./etc/ci/bootstrap-android-and-accept-licences.sh + yes | ./mach bootstrap-android --build ./mach build --android --release """) .with_artifacts( diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 0650f943278..00e54c16a88 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -79,16 +79,21 @@ class MachCommands(CommandBase): @Command('bootstrap-android', description='Install the Android SDK and NDK.', category='bootstrap') - def bootstrap_android(self): + @CommandArgument('--build', + action='store_true', + help='Install Android-specific dependencies for building') + @CommandArgument('--emulator-x86', + action='store_true', + help='Install Android x86 emulator and system image') + def bootstrap_android(self, build=False, emulator_x86=False): + if not (build or emulator_x86): + print("Must specify `--build` or `--emulator-x86` or both.") ndk = "android-ndk-r12b-{system}-{arch}" tools = "sdk-tools-{system}-4333796" - sdk_build_tools = "27.0.3" - emulator_images = [ - ("servo-arm", "25", "google_apis;armeabi-v7a"), - ("servo-x86", "28", "google_apis;x86"), - ] + emulator_platform = "android-28" + emulator_image = "system-images;%s;google_apis;x86" % emulator_platform known_sha1 = { # https://dl.google.com/android/repository/repository2-1.xml @@ -135,29 +140,34 @@ class MachCommands(CommandBase): system = platform.system().lower() machine = platform.machine().lower() arch = {"i386": "x86"}.get(machine, machine) - download("ndk", ndk.format(system=system, arch=arch), flatten=True) + if build: + download("ndk", ndk.format(system=system, arch=arch), flatten=True) download("sdk", tools.format(system=system)) - subprocess.check_call([ - path.join(toolchains, "sdk", "tools", "bin", "sdkmanager"), - "platform-tools", - "build-tools;" + sdk_build_tools, - "emulator", - ] + [ - arg - for avd_name, api_level, system_image in emulator_images - for arg in [ - "platforms;android-" + api_level, - "system-images;android-%s;%s" % (api_level, system_image), + components = [] + if emulator_x86: + components += [ + "platform-tools", + "emulator", + "platforms;" + emulator_platform, + emulator_image, ] - ]) - for avd_name, api_level, system_image in emulator_images: + if build: + components += [ + "platforms;android-18", + ] + subprocess.check_call( + [path.join(toolchains, "sdk", "tools", "bin", "sdkmanager")] + components + ) + + if emulator_x86: + avd_path = path.join(toolchains, "avd", "servo-x86") process = subprocess.Popen(stdin=subprocess.PIPE, stdout=subprocess.PIPE, args=[ path.join(toolchains, "sdk", "tools", "bin", "avdmanager"), "create", "avd", - "--path", path.join(toolchains, "avd", avd_name), - "--name", avd_name, - "--package", "system-images;android-%s;%s" % (api_level, system_image), + "--path", avd_path, + "--name", "servo-x86", + "--package", emulator_image, "--force", ]) output = b"" @@ -175,7 +185,7 @@ class MachCommands(CommandBase): if output.endswith(b"Do you wish to create a custom hardware profile? [no]"): process.stdin.write("no\n") assert process.wait() == 0 - with open(path.join(toolchains, "avd", avd_name, "config.ini"), "a") as f: + with open(path.join(avd_path, "config.ini"), "a") as f: f.write("disk.dataPartition.size=2G\n") @Command('update-hsts-preload', From 6c82c471c526f7d08613ba70659e51d248b07f82 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 11 Oct 2018 16:07:24 +0200 Subject: [PATCH 2/3] Add --accept-all-licences to ./mach bootstrap-android --- etc/ci/bootstrap-android-and-accept-licences.sh | 17 ----------------- etc/ci/buildbot_steps.yml | 8 ++++---- etc/taskcluster/decision_task.py | 2 +- python/servo/bootstrap_commands.py | 16 ++++++++++++---- 4 files changed, 17 insertions(+), 26 deletions(-) delete mode 100755 etc/ci/bootstrap-android-and-accept-licences.sh diff --git a/etc/ci/bootstrap-android-and-accept-licences.sh b/etc/ci/bootstrap-android-and-accept-licences.sh deleted file mode 100755 index 1ad22868184..00000000000 --- a/etc/ci/bootstrap-android-and-accept-licences.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -set -o errexit -set -o nounset -set -o pipefail - -# We enable pipefail above to satisfy servo-tidy, but disable it again here. -# In the case of the 'yes' program, -# exiting when the stdout pipe is broken is expected. -set +o pipefail - -cd $(dirname ${0})/../.. -yes | ./mach bootstrap-android "${@}" diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 0ceaffc7f37..6e895d30597 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -180,7 +180,7 @@ android: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - ./etc/ci/clean_build_artifacts.sh - - ./etc/ci/bootstrap-android-and-accept-licences.sh --build + - ./mach bootstrap-android --accept-all-licences --build - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --dev - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --dev - bash ./etc/ci/lockfile_changed.sh @@ -191,7 +191,7 @@ android-mac: commands: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - - ./etc/ci/bootstrap-android-and-accept-licences.sh --build + - ./mach bootstrap-android --accept-all-licences --build - ./mach build --android --dev - ./mach package --android --dev - bash ./etc/ci/lockfile_changed.sh @@ -205,7 +205,7 @@ android-x86: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - ./etc/ci/clean_build_artifacts.sh - - ./etc/ci/bootstrap-android-and-accept-licences.sh --build --emulator-x86 + - ./mach bootstrap-android --accept-all-licences --build --emulator-x86 - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target i686-linux-android --release - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach test-android-startup --release - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach test-wpt-android --release /_mozilla/mozilla/DOMParser.html /_mozilla/mozilla/webgl/context_creation_error.html @@ -220,7 +220,7 @@ android-nightly: - ./mach clean-nightlies --keep 3 --force - ./mach clean-cargo-cache --keep 3 --force - ./etc/ci/clean_build_artifacts.sh - - ./etc/ci/bootstrap-android-and-accept-licences.sh --build + - ./mach bootstrap-android --accept-all-licences --build - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --release - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --release --maven - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target=i686-linux-android --release diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index b81777aea63..58fe6ff64e1 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -87,7 +87,7 @@ def android_arm32(): # wget: servo-media-gstreamer’s build script .with_script(""" apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget - yes | ./mach bootstrap-android --build + ./mach bootstrap-android --accept-all-licences --build ./mach build --android --release """) .with_artifacts( diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 00e54c16a88..4ef19b2b45f 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -85,7 +85,10 @@ class MachCommands(CommandBase): @CommandArgument('--emulator-x86', action='store_true', help='Install Android x86 emulator and system image') - def bootstrap_android(self, build=False, emulator_x86=False): + @CommandArgument('--accept-all-licences', + action='store_true', + help='For non-interactive use') + def bootstrap_android(self, build=False, emulator_x86=False, accept_all_licences=False): if not (build or emulator_x86): print("Must specify `--build` or `--emulator-x86` or both.") @@ -156,9 +159,14 @@ class MachCommands(CommandBase): components += [ "platforms;android-18", ] - subprocess.check_call( - [path.join(toolchains, "sdk", "tools", "bin", "sdkmanager")] + components - ) + + sdkmanager = [path.join(toolchains, "sdk", "tools", "bin", "sdkmanager")] + components + if accept_all_licences: + yes = subprocess.Popen(["yes"], stdout=subprocess.PIPE) + subprocess.check_call(sdkmanager, stdin=yes.stdout) + yes.terminate() + else: + subprocess.check_call(sdkmanager) if emulator_x86: avd_path = path.join(toolchains, "avd", "servo-x86") From f0d8e8af49639678d1bd4d99855ae70737615a7a Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 11 Oct 2018 16:09:12 +0200 Subject: [PATCH 3/3] Reduce log spam when running sdkmanager --- python/servo/bootstrap_commands.py | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 4ef19b2b45f..6361fa8189a 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -163,8 +163,35 @@ class MachCommands(CommandBase): sdkmanager = [path.join(toolchains, "sdk", "tools", "bin", "sdkmanager")] + components if accept_all_licences: yes = subprocess.Popen(["yes"], stdout=subprocess.PIPE) - subprocess.check_call(sdkmanager, stdin=yes.stdout) + process = subprocess.Popen( + sdkmanager, stdin=yes.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + # Reduce progress bar spam by removing duplicate lines. + # Printing the same line again with \r is a no-op in a real terminal, + # but each line is shown individually in Taskcluster's log viewer. + previous_line = None + line = b"" + while 1: + # Read one byte at a time because in Python: + # * readline() blocks until "\n", which doesn't come before the prompt + # * read() blocks until EOF, which doesn't come before the prompt + # * read(n) keeps reading until it gets n bytes or EOF, + # but we don't know reliably how many bytes to read until the prompt + byte = process.stdout.read(1) + if len(byte) == 0: + print(line) + break + line += byte + if byte == b'\n' or byte == b'\r': + if line != previous_line: + print(line.decode("utf-8", "replace"), end="") + sys.stdout.flush() + previous_line = line + line = b"" + exit_code = process.wait() yes.terminate() + if exit_code: + return exit_code else: subprocess.check_call(sdkmanager) @@ -180,11 +207,7 @@ class MachCommands(CommandBase): ]) output = b"" while 1: - # Read one byte at a time because in Python: - # * readline() blocks until "\n", which doesn't come before the prompt - # * read() blocks until EOF, which doesn't come before the prompt - # * read(n) keeps reading until it gets n bytes or EOF, - # but we don't know reliably how many bytes to read until the prompt + # Read one byte at a time, see comment above. byte = process.stdout.read(1) if len(byte) == 0: break