Auto merge of #18942 - servo:static.rlo, r=jdm,mbrubeck

Bootstrap from more permanent URLs

The `rust-lang-ci` S3 bucket is ephemeral. `static-rust-lang-org.s3.amazonaws.com` is not going away soon, but using `static.rust-lang.org` when possible keeps things working if it ever does.

https://internals.rust-lang.org/t/updates-on-rusts-ci-uploads/6062
https://internals.rust-lang.org/t/public-stable-rust-services/6072

We’ll still need to find a solution for "alt" rustc builds. In the meantime, this is a step.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/18942)
<!-- Reviewable:end -->

python/requirements.txt

@@ -26,4 +26,9 @@ colorama == 0.3.7
 # For package uploading
 boto3 == 1.4.4
 
+# Default root CAs on Windows CI do not trust CloudFront certificates,
+# connecting to https://static.rust-lang.org would fail:
+# https://github.com/servo/servo/pull/18942
+certifi
+
 -e python/tidy

python/servo/bootstrap_commands.py

@@ -29,6 +29,7 @@ from mach.decorators import (
 import servo.bootstrap as bootstrap
 from servo.command_base import CommandBase, BIN_SUFFIX, cd
 from servo.util import delete, download_bytes, download_file, extract, host_triple
+from servo.util import STATIC_RUST_LANG_ORG_DIST, URLOPEN_KWARGS
 
 
 @CommandProvider
@@ -71,7 +72,8 @@ class MachCommands(CommandBase):
         rust_dir = path.join(self.context.sharedir, "rust", self.rust_path())
         install_dir = path.join(self.context.sharedir, "rust", self.rust_install_dir())
         version = self.rust_stable_version() if stable else "nightly"
-        static_s3 = "https://static-rust-lang-org.s3.amazonaws.com/dist"
+
+        nightly_dist = STATIC_RUST_LANG_ORG_DIST + "/" + self.rust_nightly_date()
 
         if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)):
             print("Rust compiler already downloaded.", end=" ")
@@ -87,16 +89,16 @@ class MachCommands(CommandBase):
             # giving a directory name that will be the same as the tarball name (rustc is
             # in that directory).
             if stable:
-                base_url = static_s3
+                base_url = STATIC_RUST_LANG_ORG_DIST
+            elif self.config["build"]["llvm-assertions"]:
+                base_url = nightly_dist
             else:
                 import toml
-                channel = "%s/%s/channel-rust-nightly.toml" % (static_s3, self.rust_nightly_date())
+                channel = nightly_dist + "/channel-rust-nightly.toml"
-                nightly_commit_hash = toml.load(urllib2.urlopen(channel))["pkg"]["rustc"]["git_commit_hash"]
+                manifest = toml.load(urllib2.urlopen(channel, **URLOPEN_KWARGS))
+                nightly_commit_hash = manifest["pkg"]["rustc"]["git_commit_hash"]
 
-                base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds"
+                base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds-alt/" + nightly_commit_hash
-                if not self.config["build"]["llvm-assertions"]:
-                    base_url += "-alt"
-                base_url += "/" + nightly_commit_hash
 
             rustc_url = base_url + "/rustc-%s-%s.tar.gz" % (version, host_triple())
             tgz_file = rust_dir + '-rustc.tar.gz'
@@ -131,9 +133,9 @@ class MachCommands(CommandBase):
             tarball = "rust-std-%s-%s.tar.gz" % (version, target_triple)
             tgz_file = path.join(install_dir, tarball)
             if self.use_stable_rust():
-                std_url = static_s3 + "/" + tarball
+                std_url = STATIC_RUST_LANG_ORG_DIST + "/" + tarball
             else:
-                std_url = static_s3 + "/" + self.rust_nightly_date() + "/" + tarball
+                std_url = nightly_dist + "/" + tarball
 
             download_file("Host rust library for target %s" % target_triple, std_url, tgz_file)
             print("Extracting Rust stdlib for target %s..." % target_triple)
@@ -168,8 +170,9 @@ class MachCommands(CommandBase):
         if path.isdir(docs_dir):
             shutil.rmtree(docs_dir)
         docs_name = self.rust_path().replace("rustc-", "rust-docs-")
-        docs_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s/rust-docs-nightly-%s.tar.gz"
+        docs_url = "%s/%s/rust-docs-nightly-%s.tar.gz" % (
-                    % (self.rust_nightly_date(), host_triple()))
+            STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), host_triple()
+        )
         tgz_file = path.join(rust_root, 'doc.tar.gz')
 
         download_file("Rust docs", docs_url, tgz_file)
@@ -203,8 +206,7 @@ class MachCommands(CommandBase):
         os.makedirs(cargo_dir)
 
         tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
-        nightly_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/%s/%s" % \
+        nightly_url = "%s/%s/%s" % (STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), tgz_file)
-            (self.rust_nightly_date(), tgz_file)
 
         download_file("Cargo nightly", nightly_url, tgz_file)
 

python/servo/devenv_commands.py

@@ -23,6 +23,7 @@ from mach.decorators import (
 
 from servo.command_base import CommandBase, cd, call
 from servo.build_commands import notify_build_done
+from servo.util import STATIC_RUST_LANG_ORG_DIST, URLOPEN_KWARGS
 
 
 @CommandProvider
@@ -262,8 +263,8 @@ class MachCommands(CommandBase):
              description='Update the Rust version to latest Nightly',
              category='devenv')
     def rustup(self):
-        url = "https://static-rust-lang-org.s3.amazonaws.com/dist/channel-rust-nightly-date.txt"
+        url = STATIC_RUST_LANG_ORG_DIST + "/channel-rust-nightly-date.txt"
-        nightly_date = urllib2.urlopen(url).read()
+        nightly_date = urllib2.urlopen(url, **URLOPEN_KWARGS).read()
         filename = path.join(self.context.topdir, "rust-toolchain")
         with open(filename, "w") as f:
             f.write("nightly-%s\n" % nightly_date)

python/servo/util.py

@@ -19,6 +19,21 @@ import sys
 import tarfile
 import zipfile
 import urllib2
+import certifi
+
+
+try:
+    from ssl import HAS_SNI
+except ImportError:
+    HAS_SNI = False
+
+# The cafile parameter was added in 2.7.9
+if HAS_SNI and sys.version_info >= (2, 7, 9):
+    STATIC_RUST_LANG_ORG_DIST = "https://static.rust-lang.org/dist"
+    URLOPEN_KWARGS = {"cafile": certifi.where()}
+else:
+    STATIC_RUST_LANG_ORG_DIST = "https://static-rust-lang-org.s3.amazonaws.com/dist"
+    URLOPEN_KWARGS = {}
 
 
 def delete(path):
@@ -64,16 +79,16 @@ def host_triple():
 
 def download(desc, src, writer, start_byte=0):
     if start_byte:
-        print("Resuming download of {}...".format(desc))
+        print("Resuming download of {} ...".format(src))
     else:
-        print("Downloading {}...".format(desc))
+        print("Downloading {} ...".format(src))
     dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
 
     try:
         req = urllib2.Request(src)
         if start_byte:
             req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
-        resp = urllib2.urlopen(req)
+        resp = urllib2.urlopen(req, **URLOPEN_KWARGS)
 
         fsize = None
         if resp.info().getheader('Content-Length'):

support/android/openssl.makefile

@@ -10,5 +10,5 @@ openssl-${OPENSSL_VERSION}/libssl.so: openssl-${OPENSSL_VERSION}/Configure
 	./openssl.sh ${ANDROID_NDK} ${OPENSSL_VERSION}
 
 openssl-${OPENSSL_VERSION}/Configure:
-	URL=https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/openssl-${OPENSSL_VERSION}.tar.gz; \
+	URL=https://s3.amazonaws.com/servo-deps/android-deps/openssl-${OPENSSL_VERSION}.tar.gz; \
 	curl $$URL | tar xzf -