Auto merge of #18325 - servo:rustup-toolchain, r=nox

Switch back to pinning Rust by Nightly date instead of commit hash…

… this time using a `rust-toolchain` file compatible with rustup: https://github.com/rust-lang-nursery/rustup.rs/#the-toolchain-file

And upgrade to rustc 1.21.0-nightly (c11f689d2 2017-08-29)

----

Now if both `system-rust` and `system-cargo` are set to `true` in `.servobuild`’s `[tools]` section, and the corresponding `rustc` and `cargo` binaries are in fact rustup’s wrappers, then rustup will use the correct version based on `rust-toolchain`.

CC https://github.com/servo/servo/issues/11361

Unlike https://github.com/servo/servo/pull/17927, this does not make mach use rustup directly. That should wait until https://github.com/rust-lang-nursery/rustup.rs/issues/1099 is fixed.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/18325)
<!-- Reviewable:end -->

README.md

@@ -124,7 +124,7 @@ Servo's build system automatically downloads a Rust compiler to build itself.
 This is normally a specific revision of Rust upstream, but sometimes has a
 backported patch or two.
 If you'd like to know which nightly build of Rust we use, see
-[`rust-commit-hash`](https://github.com/servo/servo/blob/master/rust-commit-hash).
+[`rust-toolchain`](https://github.com/servo/servo/blob/master/rust-toolchain).
 
 ## Building
 

appveyor.yml

@@ -36,8 +36,8 @@ branches:
     - master
 
 cache:
-  - .servo -> rust-commit-hash
+  - .servo -> rust-toolchain
-  - .cargo -> rust-commit-hash
+  - .cargo -> rust-toolchain
   - .ccache
 
 # Uncomment these lines to expose RDP access information to the build machine in the build log.

python/servo/bootstrap_commands.py

@@ -68,13 +68,10 @@ class MachCommands(CommandBase):
                      help='Use stable rustc version')
     def bootstrap_rustc(self, force=False, target=[], stable=False):
         self.set_use_stable_rust(stable)
-        version = self.rust_version()
+        rust_dir = path.join(self.context.sharedir, "rust", self.rust_path())
-        rust_path = self.rust_path()
+        install_dir = path.join(self.context.sharedir, "rust", self.rust_install_dir())
-        rust_dir = path.join(self.context.sharedir, "rust", rust_path)
+        version = self.rust_stable_version() if stable else "nightly"
-        install_dir = path.join(self.context.sharedir, "rust", version)
+        static_s3 = "https://static-rust-lang-org.s3.amazonaws.com/dist"
-        if not self.config["build"]["llvm-assertions"]:
-            if not self.use_stable_rust():
-                install_dir += "-alt"
 
         if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)):
             print("Rust compiler already downloaded.", end=" ")
@@ -90,16 +87,23 @@ class MachCommands(CommandBase):
             # giving a directory name that will be the same as the tarball name (rustc is
             # in that directory).
             if stable:
-                tarball = "rustc-%s-%s.tar.gz" % (version, host_triple())
+                base_url = static_s3
-                rustc_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/" + tarball
             else:
-                tarball = "%s/rustc-nightly-%s.tar.gz" % (version, host_triple())
+                import toml
+                import re
+                channel = "%s/%s/channel-rust-nightly.toml" % (static_s3, self.rust_nightly_date())
+                version_string = toml.load(urllib2.urlopen(channel))["pkg"]["rustc"]["version"]
+                short_commit = re.search("\(([0-9a-f]+) ", version_string).group(1)
+                commit_api = "https://api.github.com/repos/rust-lang/rust/commits/" + short_commit
+                nightly_commit_hash = json.load(urllib2.urlopen(commit_api))["sha"]
+
                 base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds"
                 if not self.config["build"]["llvm-assertions"]:
                     base_url += "-alt"
-                rustc_url = base_url + "/" + tarball
+                base_url += "/" + nightly_commit_hash
-            tgz_file = rust_dir + '-rustc.tar.gz'
 
+            rustc_url = base_url + "/rustc-%s-%s.tar.gz" % (version, host_triple())
+            tgz_file = rust_dir + '-rustc.tar.gz'
             download_file("Rust compiler", rustc_url, tgz_file)
 
             print("Extracting Rust compiler...")
@@ -111,10 +115,8 @@ class MachCommands(CommandBase):
         # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
         # This `lib` directory needs to be extracted and merged with the `rustc/lib`
         # directory from the host compiler above.
-        nightly_suffix = "" if stable else "-nightly"
-        stable_version = "-{}".format(version) if stable else ""
         lib_dir = path.join(install_dir,
-                            "rustc{}{}-{}".format(nightly_suffix, stable_version, host_triple()),
+                            "rustc-%s-%s" % (version, host_triple()),
                             "rustc", "lib", "rustlib")
 
         # ensure that the libs for the host's target is downloaded
@@ -130,26 +132,25 @@ class MachCommands(CommandBase):
                 print("Use |bootstrap-rust --force| to download again.")
                 continue
 
+            tarball = "rust-std-%s-%s.tar.gz" % (version, target_triple)
+            tgz_file = path.join(install_dir, tarball)
             if self.use_stable_rust():
-                std_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-std-%s-%s.tar.gz"
+                std_url = static_s3 + "/" + tarball
-                           % (version, target_triple))
-                tgz_file = install_dir + ('rust-std-%s-%s.tar.gz' % (version, target_triple))
             else:
-                std_url = ("https://s3.amazonaws.com/rust-lang-ci/rustc-builds/%s/rust-std-nightly-%s.tar.gz"
+                std_url = static_s3 + "/" + self.rust_nightly_date() + "/" + tarball
-                           % (version, target_triple))
-                tgz_file = install_dir + ('rust-std-nightly-%s.tar.gz' % target_triple)
 
             download_file("Host rust library for target %s" % target_triple, std_url, tgz_file)
             print("Extracting Rust stdlib for target %s..." % target_triple)
             extract(tgz_file, install_dir)
             shutil.copytree(path.join(install_dir,
-                                      "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple),
+                                      "rust-std-%s-%s" % (version, target_triple),
-                                      "rust-std-%s" % target_triple, "lib", "rustlib", target_triple),
+                                      "rust-std-%s" % target_triple,
+                                      "lib", "rustlib", target_triple),
                             path.join(install_dir,
-                                      "rustc%s%s-%s" % (nightly_suffix, stable_version, host_triple()),
+                                      "rustc-%s-%s" % (version, host_triple()),
-                                      "rustc", "lib", "rustlib", target_triple))
+                                      "rustc",
-            shutil.rmtree(path.join(install_dir,
+                                      "lib", "rustlib", target_triple))
-                          "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple)))
+            shutil.rmtree(path.join(install_dir, "rust-std-%s-%s" % (version, target_triple)))
 
             print("Rust {} libs ready.".format(target_triple))
 
@@ -171,8 +172,8 @@ class MachCommands(CommandBase):
         if path.isdir(docs_dir):
             shutil.rmtree(docs_dir)
         docs_name = self.rust_path().replace("rustc-", "rust-docs-")
-        docs_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-docs-nightly-%s.tar.gz"
+        docs_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s/rust-docs-nightly-%s.tar.gz"
-                    % host_triple())
+                    % (self.rust_nightly_date(), host_triple()))
         tgz_file = path.join(rust_root, 'doc.tar.gz')
 
         download_file("Rust docs", docs_url, tgz_file)
@@ -195,8 +196,7 @@ class MachCommands(CommandBase):
                      action='store_true',
                      help='Force download even if cargo already exists')
     def bootstrap_cargo(self, force=False):
-        cargo_dir = path.join(self.context.sharedir, "cargo",
+        cargo_dir = path.join(self.context.sharedir, "cargo", self.rust_nightly_date())
-                              self.cargo_build_id())
         if not force and path.exists(path.join(cargo_dir, "cargo", "bin", "cargo" + BIN_SUFFIX)):
             print("Cargo already downloaded.", end=" ")
             print("Use |bootstrap-cargo --force| to download again.")
@@ -207,8 +207,8 @@ class MachCommands(CommandBase):
         os.makedirs(cargo_dir)
 
         tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
-        nightly_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds/%s/%s" % \
+        nightly_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/%s/%s" % \
-            (self.cargo_build_id()[len("rust-"):], tgz_file)
+            (self.rust_nightly_date(), tgz_file)
 
         download_file("Cargo nightly", nightly_url, tgz_file)
 
@@ -291,10 +291,8 @@ class MachCommands(CommandBase):
                      default='1',
                      help='Keep up to this many most recent nightlies')
     def clean_nightlies(self, force=False, keep=None):
-        self.set_use_stable_rust(False)
+        rust_current_nightly = self.rust_nightly_date()
-        rust_current_nightly = self.rust_version()
+        rust_current_stable = self.rust_stable_version()
-        self.set_use_stable_rust(True)
-        rust_current_stable = self.rust_version()
         print("Current Rust nightly version: {}".format(rust_current_nightly))
         print("Current Rust stable version: {}".format(rust_current_stable))
         to_keep = set()
@@ -303,7 +301,7 @@ class MachCommands(CommandBase):
             to_keep.add(rust_current_nightly)
             to_keep.add(rust_current_stable)
         else:
-            for version_file in ['rust-commit-hash', 'rust-stable-version']:
+            for version_file in ['rust-toolchain', 'rust-stable-version']:
                 cmd = subprocess.Popen(
                     ['git', 'log', '--oneline', '--no-color', '-n', keep, '--patch', version_file],
                     stdout=subprocess.PIPE,

python/servo/command_base.py

@@ -285,14 +285,13 @@ class CommandBase(object):
         self.set_use_stable_rust(False)
 
     _use_stable_rust = False
-    _rust_version = None
+    _rust_stable_version = None
-    _rust_version_is_stable = False
+    _rust_nightly_date = None
-    _cargo_build_id = None
 
     def set_cargo_root(self):
         if not self.config["tools"]["system-cargo"]:
             self.config["tools"]["cargo-root"] = path.join(
-                self.context.sharedir, "cargo", self.cargo_build_id())
+                self.context.sharedir, "cargo", self.rust_nightly_date())
 
     def set_use_stable_rust(self, use_stable_rust=True):
         self._use_stable_rust = use_stable_rust
@@ -308,28 +307,39 @@ class CommandBase(object):
     def use_stable_rust(self):
         return self._use_stable_rust
 
-    def rust_path(self):
+    def rust_install_dir(self):
-        version = self.rust_version()
         if self._use_stable_rust:
-            return os.path.join(version, "rustc-%s-%s" % (version, host_triple()))
+            return self.rust_stable_version()
-        if not self.config["build"]["llvm-assertions"]:
+        elif not self.config["build"]["llvm-assertions"]:
-            version += "-alt"
+            return self.rust_nightly_date() + "-alt"
-        return os.path.join(version, "rustc-nightly-%s" % (host_triple()))
+        else:
+            return self.rust_nightly_date()
 
-    def rust_version(self):
+    def rust_path(self):
-        if self._rust_version is None or self._use_stable_rust != self._rust_version_is_stable:
+        if self._use_stable_rust:
-            filename = path.join(self.context.topdir,
+            version = self.rust_stable_version()
-                                 "rust-stable-version" if self._use_stable_rust else "rust-commit-hash")
+        else:
-            with open(filename) as f:
+            version = "nightly"
-                self._rust_version = f.read().strip()
-        return self._rust_version
 
-    def cargo_build_id(self):
+        subdir = "rustc-%s-%s" % (version, host_triple())
-        if self._cargo_build_id is None:
+        return os.path.join(self.rust_install_dir(), subdir)
-            filename = path.join(self.context.topdir, "rust-commit-hash")
+
+    def rust_stable_version(self):
+        if self._rust_stable_version is None:
+            filename = path.join("rust-stable-version")
             with open(filename) as f:
-                self._cargo_build_id = "rust-" + f.read().strip()
+                self._rust_stable_version = f.read().strip()
-        return self._cargo_build_id
+        return self._rust_stable_version
+
+    def rust_nightly_date(self):
+        if self._rust_nightly_date is None:
+            filename = path.join(self.context.topdir, "rust-toolchain")
+            with open(filename) as f:
+                toolchain = f.read().strip()
+                prefix = "nightly-"
+                assert toolchain.startswith(prefix)
+                self._rust_nightly_date = toolchain[len(prefix):]
+        return self._rust_nightly_date
 
     def get_top_dir(self):
         return self.context.topdir
@@ -587,7 +597,7 @@ class CommandBase(object):
             Registrar.dispatch("bootstrap", context=self.context)
 
         if not (self.config['tools']['system-rust'] or (rustc_binary_exists and target_exists)):
-            print("looking for rustc at %s" % (rustc_path))
+            print("Looking for rustc at %s" % (rustc_path))
             Registrar.dispatch("bootstrap-rust", context=self.context, target=filter(None, [target]),
                                stable=self._use_stable_rust)
 

python/servo/devenv_commands.py

@@ -136,7 +136,6 @@ class MachCommands(CommandBase):
 
         if dry_run:
             import toml
-            import json
             import httplib
             import colorama
 
@@ -262,31 +261,16 @@ class MachCommands(CommandBase):
     @Command('rustup',
              description='Update the Rust version to latest Nightly',
              category='devenv')
-    @CommandArgument('--master',
+    def rustup(self):
-                     action='store_true',
+        url = "https://static.rust-lang.org/dist/channel-rust-nightly-date.txt"
-                     help='Use the latest commit of the "master" branch')
+        nightly_date = urllib2.urlopen(url).read()
-    def rustup(self, master=False):
+        filename = path.join(self.context.topdir, "rust-toolchain")
-        if master:
-            url = "https://api.github.com/repos/rust-lang/rust/git/refs/heads/master"
-            commit = json.load(urllib2.urlopen(url))["object"]["sha"]
-        else:
-            import toml
-            import re
-            url = "https://static.rust-lang.org/dist/channel-rust-nightly.toml"
-            version = toml.load(urllib2.urlopen(url))["pkg"]["rustc"]["version"]
-            short_commit = re.search("\(([0-9a-f]+) ", version).group(1)
-            url = "https://api.github.com/repos/rust-lang/rust/commits/" + short_commit
-            commit = json.load(urllib2.urlopen(url))["sha"]
-        filename = path.join(self.context.topdir, "rust-commit-hash")
         with open(filename, "w") as f:
-            f.write(commit + "\n")
+            f.write("nightly-%s\n" % nightly_date)
 
-        # Reset self.config["tools"]["rust-root"]
+        # Reset self.config["tools"]["rust-root"] and self.config["tools"]["cargo-root"]
-        self._rust_version = None
+        self._rust_nightly_date = None
         self.set_use_stable_rust(False)
-
-        # Reset self.config["tools"]["cargo-root"]
-        self._cargo_build_id = None
         self.set_cargo_root()
 
         self.fetch()

rust-commit-hash

@@ -1 +0,0 @@
-13d94d5fa8129a34f5c77a1bcd76983f5aed2434

rust-toolchain

@@ -0,0 +1 @@
+nightly-2017-08-30