From efdaa0d00ac286b4dc5ad8d31e68c98a079b8917 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Wed, 18 Oct 2017 18:24:45 +0200 Subject: [PATCH 1/6] Download "non-alt" rustc builds from more permanent URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `rust-lang-ci` S3 bucket is ephemeral: https://internals.rust-lang.org/t/updates-on-rusts-ci-uploads/6062 https://internals.rust-lang.org/t/public-stable-rust-services/6072 We’ll still need to find a solution for "alt" rustc builds. In the meantime, this is a step. --- python/servo/bootstrap_commands.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 5d09fadb806..bb5e122dc48 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -88,15 +88,14 @@ class MachCommands(CommandBase): # in that directory). if stable: base_url = static_s3 - else: + elif not self.config["build"]["llvm-assertions"]: import toml channel = "%s/%s/channel-rust-nightly.toml" % (static_s3, self.rust_nightly_date()) nightly_commit_hash = toml.load(urllib2.urlopen(channel))["pkg"]["rustc"]["git_commit_hash"] - base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds" - if not self.config["build"]["llvm-assertions"]: - base_url += "-alt" - base_url += "/" + nightly_commit_hash + base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds-alt/" + nightly_commit_hash + else: + base_url = "%s/%s" % (static_s3, self.rust_nightly_date()) rustc_url = base_url + "/rustc-%s-%s.tar.gz" % (version, host_triple()) tgz_file = rust_dir + '-rustc.tar.gz' From 1f4c0c63de99ba868816351844b553f805bc1391 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Wed, 18 Oct 2017 22:45:12 +0200 Subject: [PATCH 2/6] Bootstrap: print URLs being downloaded. --- python/servo/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/servo/util.py b/python/servo/util.py index 0a74f9fe7f1..30de9fba847 100644 --- a/python/servo/util.py +++ b/python/servo/util.py @@ -64,9 +64,9 @@ def host_triple(): def download(desc, src, writer, start_byte=0): if start_byte: - print("Resuming download of {}...".format(desc)) + print("Resuming download of {} ...".format(src)) else: - print("Downloading {}...".format(desc)) + print("Downloading {} ...".format(src)) dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty()) try: From 664f2829a9d7e8f5a158111b80e71eb8b830f283 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Wed, 18 Oct 2017 22:50:20 +0200 Subject: [PATCH 3/6] Bootstrap from official static.rust-lang.org when SNI is available --- python/servo/bootstrap_commands.py | 27 ++++++++++++++------------- python/servo/command_base.py | 10 ++++++++++ python/servo/devenv_commands.py | 4 ++-- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index bb5e122dc48..46328563247 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -27,7 +27,7 @@ from mach.decorators import ( ) import servo.bootstrap as bootstrap -from servo.command_base import CommandBase, BIN_SUFFIX, cd +from servo.command_base import CommandBase, BIN_SUFFIX, cd, STATIC_RUST_LANG_ORG_DIST from servo.util import delete, download_bytes, download_file, extract, host_triple @@ -71,7 +71,8 @@ class MachCommands(CommandBase): rust_dir = path.join(self.context.sharedir, "rust", self.rust_path()) install_dir = path.join(self.context.sharedir, "rust", self.rust_install_dir()) version = self.rust_stable_version() if stable else "nightly" - static_s3 = "https://static-rust-lang-org.s3.amazonaws.com/dist" + + nightly_dist = STATIC_RUST_LANG_ORG_DIST + "/" + self.rust_nightly_date() if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)): print("Rust compiler already downloaded.", end=" ") @@ -87,15 +88,15 @@ class MachCommands(CommandBase): # giving a directory name that will be the same as the tarball name (rustc is # in that directory). if stable: - base_url = static_s3 - elif not self.config["build"]["llvm-assertions"]: + base_url = STATIC_RUST_LANG_ORG_DIST + elif self.config["build"]["llvm-assertions"]: + base_url = nightly_dist + else: import toml - channel = "%s/%s/channel-rust-nightly.toml" % (static_s3, self.rust_nightly_date()) + channel = nightly_dist + "/channel-rust-nightly.toml" nightly_commit_hash = toml.load(urllib2.urlopen(channel))["pkg"]["rustc"]["git_commit_hash"] base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds-alt/" + nightly_commit_hash - else: - base_url = "%s/%s" % (static_s3, self.rust_nightly_date()) rustc_url = base_url + "/rustc-%s-%s.tar.gz" % (version, host_triple()) tgz_file = rust_dir + '-rustc.tar.gz' @@ -130,9 +131,9 @@ class MachCommands(CommandBase): tarball = "rust-std-%s-%s.tar.gz" % (version, target_triple) tgz_file = path.join(install_dir, tarball) if self.use_stable_rust(): - std_url = static_s3 + "/" + tarball + std_url = STATIC_RUST_LANG_ORG_DIST + "/" + tarball else: - std_url = static_s3 + "/" + self.rust_nightly_date() + "/" + tarball + std_url = nightly_dist + "/" + tarball download_file("Host rust library for target %s" % target_triple, std_url, tgz_file) print("Extracting Rust stdlib for target %s..." % target_triple) @@ -167,8 +168,9 @@ class MachCommands(CommandBase): if path.isdir(docs_dir): shutil.rmtree(docs_dir) docs_name = self.rust_path().replace("rustc-", "rust-docs-") - docs_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s/rust-docs-nightly-%s.tar.gz" - % (self.rust_nightly_date(), host_triple())) + docs_url = "%s/%s/rust-docs-nightly-%s.tar.gz" % ( + STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), host_triple() + ) tgz_file = path.join(rust_root, 'doc.tar.gz') download_file("Rust docs", docs_url, tgz_file) @@ -202,8 +204,7 @@ class MachCommands(CommandBase): os.makedirs(cargo_dir) tgz_file = "cargo-nightly-%s.tar.gz" % host_triple() - nightly_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/%s/%s" % \ - (self.rust_nightly_date(), tgz_file) + nightly_url = "%s/%s/%s" % (STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), tgz_file) download_file("Cargo nightly", nightly_url, tgz_file) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 77bb88886fa..1eaf09411e3 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -25,6 +25,16 @@ import toml from servo.packages import WINDOWS_MSVC as msvc_deps from servo.util import host_triple, host_platform +try: + from ssl import HAS_SNI +except ImportError: + HAS_SNI = False + +if HAS_SNI: + STATIC_RUST_LANG_ORG_DIST = "https://static.rust-lang.org/dist" +else: + STATIC_RUST_LANG_ORG_DIST = "https://static-rust-lang-org.s3.amazonaws.com/dist" + BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py index 5e951bec8cd..8f51cfcc44d 100644 --- a/python/servo/devenv_commands.py +++ b/python/servo/devenv_commands.py @@ -21,7 +21,7 @@ from mach.decorators import ( Command, ) -from servo.command_base import CommandBase, cd, call +from servo.command_base import CommandBase, cd, call, STATIC_RUST_LANG_ORG_DIST from servo.build_commands import notify_build_done @@ -262,7 +262,7 @@ class MachCommands(CommandBase): description='Update the Rust version to latest Nightly', category='devenv') def rustup(self): - url = "https://static-rust-lang-org.s3.amazonaws.com/dist/channel-rust-nightly-date.txt" + url = STATIC_RUST_LANG_ORG_DIST + "/channel-rust-nightly-date.txt" nightly_date = urllib2.urlopen(url).read() filename = path.join(self.context.topdir, "rust-toolchain") with open(filename, "w") as f: From a931d3b840d25bf536eeae5e457b02f9e338feaa Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 19 Oct 2017 00:47:19 +0200 Subject: [PATCH 4/6] Use servo-deps AWS S3 bucket instead of rust-lang-ci for OpenSSL --- support/android/openssl.makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/android/openssl.makefile b/support/android/openssl.makefile index cfa720d2c3a..1e3ec330b64 100644 --- a/support/android/openssl.makefile +++ b/support/android/openssl.makefile @@ -10,5 +10,5 @@ openssl-${OPENSSL_VERSION}/libssl.so: openssl-${OPENSSL_VERSION}/Configure ./openssl.sh ${ANDROID_NDK} ${OPENSSL_VERSION} openssl-${OPENSSL_VERSION}/Configure: - URL=https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/openssl-${OPENSSL_VERSION}.tar.gz; \ + URL=https://s3.amazonaws.com/servo-deps/android-deps/openssl-${OPENSSL_VERSION}.tar.gz; \ curl $$URL | tar xzf - From aa62942fbd836279da8ea41d2e5d4ca9dd61fad3 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 19 Oct 2017 18:43:36 +0200 Subject: [PATCH 5/6] Bootstrap: use 'certifi' package to obtain up-to-date root CA list. The default root CA store on our Windows CI builders does not trust static.rust-lang.org --- python/requirements.txt | 5 +++++ python/servo/util.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index 44ee760eefd..0700db2a21f 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -26,4 +26,9 @@ colorama == 0.3.7 # For package uploading boto3 == 1.4.4 +# Default root CAs on Windows CI do not trust CloudFront certificates, +# connecting to https://static.rust-lang.org would fail: +# https://github.com/servo/servo/pull/18942 +certifi + -e python/tidy diff --git a/python/servo/util.py b/python/servo/util.py index 30de9fba847..485637b81f1 100644 --- a/python/servo/util.py +++ b/python/servo/util.py @@ -19,6 +19,7 @@ import sys import tarfile import zipfile import urllib2 +import certifi def delete(path): @@ -73,7 +74,7 @@ def download(desc, src, writer, start_byte=0): req = urllib2.Request(src) if start_byte: req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)}) - resp = urllib2.urlopen(req) + resp = urllib2.urlopen(req, cafile=certifi.where()) fsize = None if resp.info().getheader('Content-Length'): From 404c722920043caf548bf10c0242aac5d22a96b3 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 19 Oct 2017 19:56:18 +0200 Subject: [PATCH 6/6] Only pass cafile argument to urlopen in Python versions that support it. --- python/servo/bootstrap_commands.py | 6 ++++-- python/servo/command_base.py | 10 ---------- python/servo/devenv_commands.py | 5 +++-- python/servo/util.py | 16 +++++++++++++++- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 46328563247..aa64060ecaa 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -27,8 +27,9 @@ from mach.decorators import ( ) import servo.bootstrap as bootstrap -from servo.command_base import CommandBase, BIN_SUFFIX, cd, STATIC_RUST_LANG_ORG_DIST +from servo.command_base import CommandBase, BIN_SUFFIX, cd from servo.util import delete, download_bytes, download_file, extract, host_triple +from servo.util import STATIC_RUST_LANG_ORG_DIST, URLOPEN_KWARGS @CommandProvider @@ -94,7 +95,8 @@ class MachCommands(CommandBase): else: import toml channel = nightly_dist + "/channel-rust-nightly.toml" - nightly_commit_hash = toml.load(urllib2.urlopen(channel))["pkg"]["rustc"]["git_commit_hash"] + manifest = toml.load(urllib2.urlopen(channel, **URLOPEN_KWARGS)) + nightly_commit_hash = manifest["pkg"]["rustc"]["git_commit_hash"] base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds-alt/" + nightly_commit_hash diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 1eaf09411e3..77bb88886fa 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -25,16 +25,6 @@ import toml from servo.packages import WINDOWS_MSVC as msvc_deps from servo.util import host_triple, host_platform -try: - from ssl import HAS_SNI -except ImportError: - HAS_SNI = False - -if HAS_SNI: - STATIC_RUST_LANG_ORG_DIST = "https://static.rust-lang.org/dist" -else: - STATIC_RUST_LANG_ORG_DIST = "https://static-rust-lang-org.s3.amazonaws.com/dist" - BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py index 8f51cfcc44d..e6cb91bd20a 100644 --- a/python/servo/devenv_commands.py +++ b/python/servo/devenv_commands.py @@ -21,8 +21,9 @@ from mach.decorators import ( Command, ) -from servo.command_base import CommandBase, cd, call, STATIC_RUST_LANG_ORG_DIST +from servo.command_base import CommandBase, cd, call from servo.build_commands import notify_build_done +from servo.util import STATIC_RUST_LANG_ORG_DIST, URLOPEN_KWARGS @CommandProvider @@ -263,7 +264,7 @@ class MachCommands(CommandBase): category='devenv') def rustup(self): url = STATIC_RUST_LANG_ORG_DIST + "/channel-rust-nightly-date.txt" - nightly_date = urllib2.urlopen(url).read() + nightly_date = urllib2.urlopen(url, **URLOPEN_KWARGS).read() filename = path.join(self.context.topdir, "rust-toolchain") with open(filename, "w") as f: f.write("nightly-%s\n" % nightly_date) diff --git a/python/servo/util.py b/python/servo/util.py index 485637b81f1..78158b840da 100644 --- a/python/servo/util.py +++ b/python/servo/util.py @@ -22,6 +22,20 @@ import urllib2 import certifi +try: + from ssl import HAS_SNI +except ImportError: + HAS_SNI = False + +# The cafile parameter was added in 2.7.9 +if HAS_SNI and sys.version_info >= (2, 7, 9): + STATIC_RUST_LANG_ORG_DIST = "https://static.rust-lang.org/dist" + URLOPEN_KWARGS = {"cafile": certifi.where()} +else: + STATIC_RUST_LANG_ORG_DIST = "https://static-rust-lang-org.s3.amazonaws.com/dist" + URLOPEN_KWARGS = {} + + def delete(path): if os.path.isdir(path) and not os.path.islink(path): shutil.rmtree(path) @@ -74,7 +88,7 @@ def download(desc, src, writer, start_byte=0): req = urllib2.Request(src) if start_byte: req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)}) - resp = urllib2.urlopen(req, cafile=certifi.where()) + resp = urllib2.urlopen(req, **URLOPEN_KWARGS) fsize = None if resp.info().getheader('Content-Length'):