Auto merge of #19395 - servo:rustup.rs, r=jdm

Use rustup.rs instead of custom bootstrap

Fixes #11361, closes #18874, fixes #19365.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/19395)
<!-- Reviewable:end -->

.travis.yml

@@ -19,6 +19,8 @@ matrix:
         - sudo apt-get install clang-3.9 llvm-3.9 llvm-3.9-runtime -y
         - export LLVM_CONFIG=/usr/lib/llvm-3.9/bin/llvm-config
         - export CC=gcc-5 CXX=g++-5
+        - curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y
+        - source ~/.profile
       script:
          - ./mach build -d --verbose
          - ./mach test-unit
@@ -28,8 +30,9 @@ matrix:
          - bash etc/ci/lockfile_changed.sh
       cache:
         directories:
-          - .cargo
           - .servo
+          - $HOME/.cargo
+          - $HOME/.rustup
           - $HOME/.ccache
       before_cache:
         - ./mach clean-nightlies --keep 2 --force

README.md

@@ -14,6 +14,32 @@ Visit the [Servo Project page](https://servo.org/) for news and guides.
 
 ## Setting up your environment
 
+### Rustup.rs
+
+Building servo requires [rustup](https://rustup.rs/), version 1.8.0 or more recent.
+If you have an older version, run `rustup self update`.
+
+To install on Windows, download and run [`rustup-init.exe`](https://win.rustup.rs/)
+then follow the onscreen instructions.
+
+To install on other systems, run:
+
+```sh
+curl https://sh.rustup.rs -sSf | sh
+```
+
+This will also download the current stable version of Rust, which Servo won’t use.
+To skip that step, run instead:
+
+```
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none
+```
+
+See also [Other installation methods](
+https://github.com/rust-lang-nursery/rustup.rs/#other-installation-methods)
+
+### Other dependencies
+
 Please select your operating system:
 * [OS X](#os-x)
 * [Debian-based Linuxes](#on-debian-based-linuxes)
@@ -152,11 +178,9 @@ Pre-installed Android tools are needed. See wiki for
 
 ## The Rust compiler
 
-Servo's build system automatically downloads a Rust compiler to build itself.
+Servo's build system uses rustup.rs to automatically download a Rust compiler.
-This is normally a specific revision of Rust upstream, but sometimes has a
+This is a specific version of Rust Nightly determined by the
-backported patch or two.
+[`rust-toolchain`](https://github.com/servo/servo/blob/master/rust-toolchain) file.
-If you'd like to know which nightly build of Rust we use, see
-[`rust-toolchain`](https://github.com/servo/servo/blob/master/rust-toolchain).
 
 ## Building
 

appveyor.yml

@@ -37,10 +37,19 @@ branches:
     - master
 
 cache:
+  - C:\Users\appveyor\.rustup -> rust-toolchain
+  - C:\Users\appveyor\.cargo -> rust-toolchain
   - .servo -> rust-toolchain
-  - .cargo -> rust-toolchain
   - .ccache
 
+install:
+  - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain none
+  - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
+  - rustup -V
+  - mach rustc --version
+  - mach cargo --version
+
 # Uncomment these lines to expose RDP access information to the build machine in the build log.
 #init:
 #  - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

etc/rustdoc-with-private

@@ -0,0 +1,4 @@
+#!/bin/sh
+# Skip the strip-private and strip-hidden rustdoc passes
+# https://github.com/rust-lang/rust/issues/15347
+rustdoc --no-defaults --passes collapse-docs --passes unindent-comments --passes strip-priv-imports "$@"

python/servo/bootstrap_commands.py

@@ -14,7 +14,6 @@ import json
 import os
 import os.path as path
 import re
-import shutil
 import subprocess
 import sys
 import urllib2
@@ -27,9 +26,8 @@ from mach.decorators import (
 )
 
 import servo.bootstrap as bootstrap
-from servo.command_base import CommandBase, BIN_SUFFIX, cd
+from servo.command_base import CommandBase, cd, check_call
-from servo.util import delete, download_bytes, download_file, extract, host_triple
+from servo.util import delete, download_bytes
-from servo.util import STATIC_RUST_LANG_ORG_DIST
 
 
 @CommandProvider
@@ -39,6 +37,7 @@ class MachCommands(CommandBase):
              category='bootstrap')
     def env(self):
         env = self.build_env()
+        print("export RUSTFLAGS=%s" % env["RUSTFLAGS"])
         print("export PATH=%s" % env["PATH"])
         if sys.platform == "darwin":
             print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
@@ -54,161 +53,6 @@ class MachCommands(CommandBase):
     def bootstrap(self, force=False):
         return bootstrap.bootstrap(self.context, force=force)
 
-    @Command('bootstrap-rust',
-             description='Download the Rust compiler',
-             category='bootstrap')
-    @CommandArgument('--force', '-f',
-                     action='store_true',
-                     help='Force download even if a copy already exists')
-    @CommandArgument('--target',
-                     action='append',
-                     default=[],
-                     help='Download rust stdlib for specified target')
-    @CommandArgument('--stable',
-                     action='store_true',
-                     help='Use stable rustc version')
-    def bootstrap_rustc(self, force=False, target=[], stable=False):
-        self.set_use_stable_rust(stable)
-        rust_dir = path.join(self.context.sharedir, "rust", self.rust_path())
-        install_dir = path.join(self.context.sharedir, "rust", self.rust_install_dir())
-        version = self.rust_stable_version() if stable else "nightly"
-
-        nightly_dist = STATIC_RUST_LANG_ORG_DIST + "/" + self.rust_nightly_date()
-
-        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)):
-            print("Rust compiler already downloaded.", end=" ")
-            print("Use |bootstrap-rust --force| to download again.")
-        else:
-            if path.isdir(rust_dir):
-                shutil.rmtree(rust_dir)
-            os.makedirs(rust_dir)
-
-            # The nightly Rust compiler is hosted on the nightly server under the date with a name
-            # rustc-nightly-HOST-TRIPLE.tar.gz, whereas the stable compiler is named
-            # rustc-VERSION-HOST-TRIPLE.tar.gz. We just need to pull down and extract it,
-            # giving a directory name that will be the same as the tarball name (rustc is
-            # in that directory).
-            if stable:
-                base_url = STATIC_RUST_LANG_ORG_DIST
-            else:
-                base_url = nightly_dist
-
-            rustc_url = base_url + "/rustc-%s-%s.tar.gz" % (version, host_triple())
-            tgz_file = rust_dir + '-rustc.tar.gz'
-            download_file("Rust compiler", rustc_url, tgz_file)
-
-            print("Extracting Rust compiler...")
-            extract(tgz_file, install_dir)
-            print("Rust compiler ready.")
-
-        # Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz` for the nightly
-        # releases, or rust-std-VERSION-TRIPLE.tar.gz for stable releases, with
-        # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
-        # This `lib` directory needs to be extracted and merged with the `rustc/lib`
-        # directory from the host compiler above.
-        lib_dir = path.join(install_dir,
-                            "rustc-%s-%s" % (version, host_triple()),
-                            "rustc", "lib", "rustlib")
-
-        # ensure that the libs for the host's target is downloaded
-        host_target = host_triple()
-        if host_target not in target:
-            target.append(host_target)
-
-        for target_triple in target:
-            target_lib_dir = path.join(lib_dir, target_triple)
-            if path.exists(target_lib_dir):
-                # No need to check for force. If --force the directory is already deleted
-                print("Rust lib for target {} already downloaded.".format(target_triple), end=" ")
-                print("Use |bootstrap-rust --force| to download again.")
-                continue
-
-            tarball = "rust-std-%s-%s.tar.gz" % (version, target_triple)
-            tgz_file = path.join(install_dir, tarball)
-            if self.use_stable_rust():
-                std_url = STATIC_RUST_LANG_ORG_DIST + "/" + tarball
-            else:
-                std_url = nightly_dist + "/" + tarball
-
-            download_file("Host rust library for target %s" % target_triple, std_url, tgz_file)
-            print("Extracting Rust stdlib for target %s..." % target_triple)
-            extract(tgz_file, install_dir)
-            shutil.copytree(path.join(install_dir,
-                                      "rust-std-%s-%s" % (version, target_triple),
-                                      "rust-std-%s" % target_triple,
-                                      "lib", "rustlib", target_triple),
-                            path.join(install_dir,
-                                      "rustc-%s-%s" % (version, host_triple()),
-                                      "rustc",
-                                      "lib", "rustlib", target_triple))
-            shutil.rmtree(path.join(install_dir, "rust-std-%s-%s" % (version, target_triple)))
-
-            print("Rust {} libs ready.".format(target_triple))
-
-    @Command('bootstrap-rust-docs',
-             description='Download the Rust documentation',
-             category='bootstrap')
-    @CommandArgument('--force', '-f',
-                     action='store_true',
-                     help='Force download even if docs already exist')
-    def bootstrap_rustc_docs(self, force=False):
-        self.ensure_bootstrapped()
-        rust_root = self.config["tools"]["rust-root"]
-        docs_dir = path.join(rust_root, "doc")
-        if not force and path.exists(docs_dir):
-            print("Rust docs already downloaded.", end=" ")
-            print("Use |bootstrap-rust-docs --force| to download again.")
-            return
-
-        if path.isdir(docs_dir):
-            shutil.rmtree(docs_dir)
-        docs_name = self.rust_path().replace("rustc-", "rust-docs-")
-        docs_url = "%s/%s/rust-docs-nightly-%s.tar.gz" % (
-            STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), host_triple()
-        )
-        tgz_file = path.join(rust_root, 'doc.tar.gz')
-
-        download_file("Rust docs", docs_url, tgz_file)
-
-        print("Extracting Rust docs...")
-        temp_dir = path.join(rust_root, "temp_docs")
-        if path.isdir(temp_dir):
-            shutil.rmtree(temp_dir)
-        extract(tgz_file, temp_dir)
-        shutil.move(path.join(temp_dir, docs_name.split("/")[1],
-                              "rust-docs", "share", "doc", "rust", "html"),
-                    docs_dir)
-        shutil.rmtree(temp_dir)
-        print("Rust docs ready.")
-
-    @Command('bootstrap-cargo',
-             description='Download the Cargo build tool',
-             category='bootstrap')
-    @CommandArgument('--force', '-f',
-                     action='store_true',
-                     help='Force download even if cargo already exists')
-    def bootstrap_cargo(self, force=False):
-        cargo_dir = path.join(self.context.sharedir, "cargo", self.rust_nightly_date())
-        if not force and path.exists(path.join(cargo_dir, "cargo", "bin", "cargo" + BIN_SUFFIX)):
-            print("Cargo already downloaded.", end=" ")
-            print("Use |bootstrap-cargo --force| to download again.")
-            return
-
-        if path.isdir(cargo_dir):
-            shutil.rmtree(cargo_dir)
-        os.makedirs(cargo_dir)
-
-        tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
-        nightly_url = "%s/%s/%s" % (STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), tgz_file)
-
-        download_file("Cargo nightly", nightly_url, tgz_file)
-
-        print("Extracting Cargo nightly...")
-        nightly_dir = path.join(cargo_dir,
-                                path.basename(tgz_file).replace(".tar.gz", ""))
-        extract(tgz_file, cargo_dir, movedir=nightly_dir)
-        print("Cargo ready.")
-
     @Command('update-hsts-preload',
              description='Download the HSTS preload list',
              category='bootstrap')
@@ -282,51 +126,31 @@ class MachCommands(CommandBase):
                      default='1',
                      help='Keep up to this many most recent nightlies')
     def clean_nightlies(self, force=False, keep=None):
-        rust_current_nightly = self.rust_nightly_date()
+        default_toolchain = self.default_toolchain()
-        rust_current_stable = self.rust_stable_version()
+        geckolib_toolchain = self.geckolib_toolchain()
-        print("Current Rust nightly version: {}".format(rust_current_nightly))
+        print("Current Rust version for Servo: {}".format(default_toolchain))
-        print("Current Rust stable version: {}".format(rust_current_stable))
+        print("Current Rust version for geckolib: {}".format(geckolib_toolchain))
-        to_keep = set()
+        old_toolchains = []
-        if int(keep) == 1:
+        keep = int(keep)
-            # Optimize keep=1 case to not invoke git
+        for toolchain_file in ['rust-toolchain', 'geckolib-rust-toolchain']:
-            to_keep.add(rust_current_nightly)
+            stdout = subprocess.check_output(['git', 'log', '--format=%H', toolchain_file])
-            to_keep.add(rust_current_stable)
+            for i, commit_hash in enumerate(stdout.split(), 1):
-        else:
+                if i > keep:
-            for version_file in ['rust-toolchain', 'rust-stable-version']:
+                    toolchain = subprocess.check_output(
-                cmd = subprocess.Popen(
+                        ['git', 'show', '%s:%s' % (commit_hash, toolchain_file)])
-                    ['git', 'log', '--oneline', '--no-color', '-n', keep, '--patch', version_file],
+                    old_toolchains.append(toolchain.strip())
-                    stdout=subprocess.PIPE,
-                    universal_newlines=True
-                )
-                stdout, _ = cmd.communicate()
-                for line in stdout.splitlines():
-                    if line.startswith(b"+") and not line.startswith(b"+++"):
-                        line = line[len(b"+"):]
-                        if line.startswith(b"nightly-"):
-                            line = line[len(b"nightly-"):]
-                        to_keep.add(line)
 
         removing_anything = False
-        for tool in ["rust", "cargo"]:
+        stdout = subprocess.check_output(['rustup', 'toolchain', 'list'])
-            base = path.join(self.context.sharedir, tool)
+        for toolchain_with_host in stdout.split():
-            if not path.isdir(base):
+            for old in old_toolchains:
-                continue
+                if toolchain_with_host.startswith(old):
-            for name in os.listdir(base):
-                full_path = path.join(base, name)
-                if name.startswith("rust-"):
-                    name = name[len("rust-"):]
-                if name.endswith("-alt"):
-                    name = name[:-len("-alt")]
-                if name not in to_keep:
                     removing_anything = True
                     if force:
-                        print("Removing {}".format(full_path))
+                        print("Removing {}".format(toolchain_with_host))
-                        try:
+                        check_call(["rustup", "uninstall", toolchain_with_host])
-                            delete(full_path)
-                        except OSError as e:
-                            print("Removal failed with error {}".format(e))
                     else:
-                        print("Would remove {}".format(full_path))
+                        print("Would remove {}".format(toolchain_with_host))
         if not removing_anything:
             print("Nothing to remove.")
         elif not force:

python/servo/build_commands.py

@@ -233,6 +233,13 @@ class MachCommands(CommandBase):
             target = self.config["android"]["target"]
 
         if target:
+            if self.config["tools"]["use-rustup"]:
+                # 'rustup target add' fails if the toolchain is not installed at all.
+                self.call_rustup_run(["rustc", "--version"])
+
+                check_call(["rustup" + BIN_SUFFIX, "target", "add",
+                            "--toolchain", self.toolchain(), target])
+
             opts += ["--target", target]
             if not android:
                 android = self.handle_android_target(target)
@@ -340,10 +347,7 @@ class MachCommands(CommandBase):
                 os.makedirs(aar_out_dir)
             env["AAR_OUT_DIR"] = aar_out_dir
 
-        cargo_binary = "cargo" + BIN_SUFFIX
+        status = self.call_rustup_run(["cargo", "build"] + opts, env=env, verbose=verbose)
-
-        status = call(
-            [cargo_binary, "build"] + opts, env=env, verbose=verbose)
         elapsed = time() - build_start
 
         # Do some additional things if the build succeeded
@@ -430,9 +434,9 @@ class MachCommands(CommandBase):
             # Unlike RUSTFLAGS, these are only passed in the final rustc invocation
             # so that `./mach build` followed by `./mach build-cef` both build
             # common dependencies with the same flags.
-            opts += ["--", "-C", "link-args=-Xlinker -undefined -Xlinker dynamic_lookup"]
+            opts += ["--", "-Clink-args=-Xlinker -undefined -Xlinker dynamic_lookup"]
 
-        ret = call(["cargo", "rustc"] + opts, env=env, verbose=verbose)
+        ret = self.call_rustup_run(["cargo", "rustc"] + opts, env=env, verbose=verbose)
         elapsed = time() - build_start
 
         # Generate Desktop Notification if elapsed-time > some threshold value
@@ -455,7 +459,7 @@ class MachCommands(CommandBase):
                      action='store_true',
                      help='Build in release mode')
     def build_geckolib(self, jobs=None, verbose=False, release=False):
-        self.set_use_stable_rust()
+        self.set_use_geckolib_toolchain()
         self.ensure_bootstrapped()
         self.ensure_clobbered()
 
@@ -475,7 +479,7 @@ class MachCommands(CommandBase):
             opts += ["--features", ' '.join(features)]
 
         build_start = time()
-        ret = call(["cargo", "build"] + opts, env=env, verbose=verbose)
+        ret = self.call_rustup_run(["cargo", "build"] + opts, env=env, verbose=verbose)
         elapsed = time() - build_start
 
         # Generate Desktop Notification if elapsed-time > some threshold value

python/servo/command_base.py

@@ -7,12 +7,15 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+from errno import ENOENT as NO_SUCH_FILE_OR_DIRECTORY
 from glob import glob
 import gzip
 import itertools
 import locale
 import os
 from os import path
+import platform
+import re
 import contextlib
 import subprocess
 from subprocess import PIPE
@@ -138,6 +141,18 @@ def call(*args, **kwargs):
     return subprocess.call(*args, shell=sys.platform == 'win32', **kwargs)
 
 
+def check_output(*args, **kwargs):
+    """Wrap `subprocess.call`, printing the command if verbose=True."""
+    verbose = kwargs.pop('verbose', False)
+    if verbose:
+        print(' '.join(args[0]))
+    if 'env' in kwargs:
+        kwargs['env'] = normalize_env(kwargs['env'])
+    # we have to use shell=True in order to get PATH handling
+    # when looking for the binary on Windows
+    return subprocess.check_output(*args, shell=sys.platform == 'win32', **kwargs)
+
+
 def check_call(*args, **kwargs):
     """Wrap `subprocess.check_call`, printing the command if verbose=True.
 
@@ -254,10 +269,7 @@ class CommandBase(object):
 
         context.sharedir = self.config["tools"]["cache-dir"]
 
-        self.config["tools"].setdefault("system-rust", False)
+        self.config["tools"].setdefault("use-rustup", True)
-        self.config["tools"].setdefault("system-cargo", False)
-        self.config["tools"].setdefault("rust-root", "")
-        self.config["tools"].setdefault("cargo-root", "")
         self.config["tools"].setdefault("rustc-with-gold", get_env_bool("SERVO_RUSTC_WITH_GOLD", True))
 
         self.config.setdefault("build", {})
@@ -276,63 +288,63 @@ class CommandBase(object):
         # Set default android target
         self.handle_android_target("armv7-linux-androideabi")
 
-        self.set_cargo_root()
+        self.set_use_geckolib_toolchain(False)
-        self.set_use_stable_rust(False)
 
-    _use_stable_rust = False
+    _use_geckolib_toolchain = False
-    _rust_stable_version = None
+    _geckolib_toolchain = None
-    _rust_nightly_date = None
+    _default_toolchain = None
 
-    def set_cargo_root(self):
+    def set_use_geckolib_toolchain(self, use_geckolib_toolchain=True):
-        if not self.config["tools"]["system-cargo"]:
+        self._use_geckolib_toolchain = use_geckolib_toolchain
-            self.config["tools"]["cargo-root"] = path.join(
+        if use_geckolib_toolchain:
-                self.context.sharedir, "cargo", self.rust_nightly_date())
-
-    def set_use_stable_rust(self, use_stable_rust=True):
-        self._use_stable_rust = use_stable_rust
-        if not self.config["tools"]["system-rust"]:
-            self.config["tools"]["rust-root"] = path.join(
-                self.context.sharedir, "rust", self.rust_path())
-        if use_stable_rust:
             # Cargo maintainer's position is that CARGO_INCREMENTAL is a nightly-only feature
             # and should not be used on the stable channel.
             # https://github.com/rust-lang/cargo/issues/3835
             self.config["build"]["incremental"] = False
 
-    def use_stable_rust(self):
+    def toolchain(self):
-        return self._use_stable_rust
+        if self._use_geckolib_toolchain:
-
+            return self.geckolib_toolchain()
-    def rust_install_dir(self):
-        if self._use_stable_rust:
-            return self.rust_stable_version()
         else:
-            return self.rust_nightly_date()
+            return self.default_toolchain()
 
-    def rust_path(self):
+    def geckolib_toolchain(self):
-        if self._use_stable_rust:
+        if self._geckolib_toolchain is None:
-            version = self.rust_stable_version()
+            filename = path.join(self.context.topdir, "geckolib-rust-toolchain")
-        else:
-            version = "nightly"
-
-        subdir = "rustc-%s-%s" % (version, host_triple())
-        return os.path.join(self.rust_install_dir(), subdir)
-
-    def rust_stable_version(self):
-        if self._rust_stable_version is None:
-            filename = path.join("rust-stable-version")
             with open(filename) as f:
-                self._rust_stable_version = f.read().strip()
+                self._geckolib_toolchain = f.read().strip()
-        return self._rust_stable_version
+        return self._geckolib_toolchain
 
-    def rust_nightly_date(self):
+    def default_toolchain(self):
-        if self._rust_nightly_date is None:
+        if self._default_toolchain is None:
             filename = path.join(self.context.topdir, "rust-toolchain")
             with open(filename) as f:
-                toolchain = f.read().strip()
+                self._default_toolchain = f.read().strip()
-                prefix = "nightly-"
+        return self._default_toolchain
-                assert toolchain.startswith(prefix)
+
-                self._rust_nightly_date = toolchain[len(prefix):]
+    def call_rustup_run(self, args, **kwargs):
-        return self._rust_nightly_date
+        if self.config["tools"]["use-rustup"]:
+            try:
+                version_line = subprocess.check_output(["rustup" + BIN_SUFFIX, "--version"])
+            except OSError as e:
+                if e.errno == NO_SUCH_FILE_OR_DIRECTORY:
+                    print "It looks like rustup is not installed. See instructions at " \
+                          "https://github.com/servo/servo/#setting-up-your-environment"
+                    print
+                    return 1
+                raise
+            version = tuple(map(int, re.match("rustup (\d+)\.(\d+)\.(\d+)", version_line).groups()))
+            if version < (1, 8, 0):
+                print "rustup is at version %s.%s.%s, Servo requires 1.8.0 or more recent." % version
+                print "Try running 'rustup self update'."
+                return 1
+            toolchain = self.toolchain()
+            if platform.system() == "Windows":
+                toolchain += "-x86_64-pc-windows-msvc"
+            args = ["rustup" + BIN_SUFFIX, "run", "--install", toolchain] + args
+        else:
+            args[0] += BIN_SUFFIX
+        return call(args, **kwargs)
 
     def get_top_dir(self):
         return self.context.topdir
@@ -388,7 +400,7 @@ class CommandBase(object):
                                   " --release" if release else ""))
         sys.exit()
 
-    def build_env(self, hosts_file_path=None, target=None, is_build=False, geckolib=False):
+    def build_env(self, hosts_file_path=None, target=None, is_build=False, geckolib=False, test_unit=False):
         """Return an extended environment dictionary."""
         env = os.environ.copy()
         if sys.platform == "win32" and type(env['PATH']) == unicode:
@@ -423,29 +435,9 @@ class CommandBase(object):
             # Always build harfbuzz from source
             env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
 
-        if not self.config["tools"]["system-rust"] \
-                or self.config["tools"]["rust-root"]:
-            env["RUST_ROOT"] = self.config["tools"]["rust-root"]
-            # These paths are for when rust-root points to an unpacked installer
-            extra_path += [path.join(self.config["tools"]["rust-root"], "rustc", "bin")]
-            extra_lib += [path.join(self.config["tools"]["rust-root"], "rustc", "lib")]
-            # These paths are for when rust-root points to a rustc sysroot
-            extra_path += [path.join(self.config["tools"]["rust-root"], "bin")]
-            extra_lib += [path.join(self.config["tools"]["rust-root"], "lib")]
-
-        if not self.config["tools"]["system-cargo"] \
-                or self.config["tools"]["cargo-root"]:
-            # This path is for when rust-root points to an unpacked installer
-            extra_path += [
-                path.join(self.config["tools"]["cargo-root"], "cargo", "bin")]
-            # This path is for when rust-root points to a rustc sysroot
-            extra_path += [
-                path.join(self.config["tools"]["cargo-root"], "bin")]
-
         if extra_path:
             env["PATH"] = "%s%s%s" % (os.pathsep.join(extra_path), os.pathsep, env["PATH"])
 
-        env["CARGO_HOME"] = self.config["tools"]["cargo-home-dir"]
         if self.config["build"]["incremental"]:
             env["CARGO_INCREMENTAL"] = "1"
 
@@ -486,7 +478,10 @@ class CommandBase(object):
         if hosts_file_path:
             env['HOST_FILE'] = hosts_file_path
 
-        env['RUSTDOCFLAGS'] = "--document-private-items"
+        if not test_unit:
+            # This wrapper script is in bash and doesn't work on Windows
+            # where we want to run doctests as part of `./mach test-unit`
+            env['RUSTDOC'] = path.join(self.context.topdir, 'etc', 'rustdoc-with-private')
 
         if self.config["build"]["rustflags"]:
             env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " " + self.config["build"]["rustflags"]
@@ -588,33 +583,10 @@ class CommandBase(object):
 
         target_platform = target or host_triple()
 
-        rust_root = self.config["tools"]["rust-root"]
-        rustc_path = path.join(
-            rust_root, "rustc", "bin", "rustc" + BIN_SUFFIX
-        )
-        rustc_binary_exists = path.exists(rustc_path)
-
-        base_target_path = path.join(rust_root, "rustc", "lib", "rustlib")
-
-        target_path = path.join(base_target_path, target_platform)
-        target_exists = path.exists(target_path)
-
         # Always check if all needed MSVC dependencies are installed
         if "msvc" in target_platform:
             Registrar.dispatch("bootstrap", context=self.context)
 
-        if not (self.config['tools']['system-rust'] or (rustc_binary_exists and target_exists)):
-            print("Looking for rustc at %s" % (rustc_path))
-            Registrar.dispatch("bootstrap-rust", context=self.context, target=filter(None, [target]),
-                               stable=self._use_stable_rust)
-
-        cargo_path = path.join(self.config["tools"]["cargo-root"], "cargo", "bin",
-                               "cargo" + BIN_SUFFIX)
-        cargo_binary_exists = path.exists(cargo_path)
-
-        if not self.config["tools"]["system-cargo"] and not cargo_binary_exists:
-            Registrar.dispatch("bootstrap-cargo", context=self.context)
-
         self.context.bootstrapped = True
 
     def ensure_clobbered(self, target_dir=None):

python/servo/devenv_commands.py

@@ -21,7 +21,7 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd, call
+from servo.command_base import CommandBase, cd, call, BIN_SUFFIX
 from servo.build_commands import notify_build_done
 from servo.util import STATIC_RUST_LANG_ORG_DIST, URLOPEN_KWARGS
 
@@ -43,16 +43,16 @@ class MachCommands(CommandBase):
             # for c in $(cargo --list | tail -$(($(cargo --list | wc -l) - 1))); do
             #   (cargo help $c 2>&1 | grep "\\--package" >/dev/null 2>&1) && echo $c
             # done
-            if params[0] and params[0] in [
+            if params and params[0] in [
                 'bench', 'build', 'check', 'clean', 'doc', 'fmt', 'pkgid',
                 'run', 'rustc', 'rustdoc', 'test', 'update',
             ]:
                 params[1:1] = ['--package', 'geckoservo']
 
-            self.set_use_stable_rust()
+            self.set_use_geckolib_toolchain()
 
         build_start = time()
-        status = call(['cargo'] + params, env=env)
+        status = self.call_rustup_run(["cargo"] + params, env=env)
         elapsed = time() - build_start
 
         notify_build_done(self.config, elapsed, status == 0)
@@ -197,8 +197,7 @@ class MachCommands(CommandBase):
             self.ensure_bootstrapped()
 
             with cd(self.context.topdir):
-                call(["cargo", "update"] + params,
+                self.call_rustup_run(["cargo", "update"] + params, env=self.build_env())
-                     env=self.build_env())
 
     @Command('rustc',
              description='Run the Rust compiler',
@@ -211,8 +210,7 @@ class MachCommands(CommandBase):
             params = []
 
         self.ensure_bootstrapped()
-
+        return self.call_rustup_run(["rustc"] + params, env=self.build_env())
-        return call(["rustc"] + params, env=self.build_env())
 
     @Command('rustc-geckolib',
              description='Run the Rust compiler with the same compiler version and root crate as build-geckolib',
@@ -224,17 +222,11 @@ class MachCommands(CommandBase):
         if params is None:
             params = []
 
-        self.set_use_stable_rust()
+        self.set_use_geckolib_toolchain()
         self.ensure_bootstrapped()
         env = self.build_env(geckolib=True)
 
-        return call(["rustc"] + params, env=env)
+        return self.call_rustup_run(["rustc"] + params, env=env)
-
-    @Command('rust-root',
-             description='Print the path to the root of the Rust compiler',
-             category='devenv')
-    def rust_root(self):
-        print(self.config["tools"]["rust-root"])
 
     @Command('grep',
              description='`git grep` for selected directories.',
@@ -267,24 +259,17 @@ class MachCommands(CommandBase):
     def rustup(self):
         url = STATIC_RUST_LANG_ORG_DIST + "/channel-rust-nightly-date.txt"
         nightly_date = urllib2.urlopen(url, **URLOPEN_KWARGS).read()
+        toolchain = "nightly-" + nightly_date
         filename = path.join(self.context.topdir, "rust-toolchain")
         with open(filename, "w") as f:
-            f.write("nightly-%s\n" % nightly_date)
+            f.write(toolchain + "\n")
-
+        return call(["rustup" + BIN_SUFFIX, "install", toolchain])
-        # Reset self.config["tools"]["rust-root"] and self.config["tools"]["cargo-root"]
-        self._rust_nightly_date = None
-        self.set_use_stable_rust(False)
-        self.set_cargo_root()
-
-        self.fetch()
 
     @Command('fetch',
              description='Fetch Rust, Cargo and Cargo dependencies',
              category='devenv')
     def fetch(self):
-        # Fetch Rust and Cargo
         self.ensure_bootstrapped()
 
-        # Fetch Cargo dependencies
         with cd(self.context.topdir):
-            call(["cargo", "fetch"], env=self.build_env())
+            return self.call_rustup_run(["cargo", "fetch"], env=self.build_env())

python/servo/post_build_commands.py

@@ -14,8 +14,6 @@ import os.path as path
 import subprocess
 from shutil import copytree, rmtree, copy2
 
-from mach.registrar import Registrar
-
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
@@ -24,7 +22,7 @@ from mach.decorators import (
 
 from servo.command_base import (
     CommandBase,
-    call, check_call,
+    check_call, check_output, BIN_SUFFIX,
     is_linux, is_windows, is_macosx, set_osmesa_env,
     get_browserhtml_path,
 )
@@ -211,10 +209,14 @@ class PostBuildCommands(CommandBase):
         'params', nargs='...',
         help="Command-line arguments to be passed through to cargo doc")
     def doc(self, params):
+        env = os.environ.copy()
+        env["RUSTUP_TOOLCHAIN"] = self.toolchain()
+        rustc_path = check_output(["rustup" + BIN_SUFFIX, "which", "rustc"], env=env)
+        assert path.basename(path.dirname(rustc_path)) == "bin"
+        toolchain_path = path.dirname(path.dirname(rustc_path))
+        rust_docs = path.join(toolchain_path, "share", "doc", "rust", "html")
+
         self.ensure_bootstrapped()
-        if not path.exists(path.join(self.config["tools"]["rust-root"], "doc")):
-            Registrar.dispatch("bootstrap-rust-docs", context=self.context)
-        rust_docs = path.join(self.config["tools"]["rust-root"], "doc")
         docs = path.join(self.get_target_dir(), "doc")
         if not path.exists(docs):
             os.makedirs(docs)
@@ -234,8 +236,10 @@ class PostBuildCommands(CommandBase):
                     else:
                         copy2(full_name, destination)
 
-        return call(["cargo", "doc"] + params,
+        return self.call_rustup_run(
-                    env=self.build_env(), cwd=self.servo_crate())
+            ["cargo", "doc", "--manifest-path", self.servo_manifest()] + params,
+            env=self.build_env()
+        )
 
     @Command('browse-doc',
              description='Generate documentation and open it in a web browser',

python/servo/testing_commands.py

@@ -34,7 +34,7 @@ from mach.decorators import (
 
 from servo.command_base import (
     BuildNotFound, CommandBase,
-    call, cd, check_call, set_osmesa_env,
+    call, check_call, set_osmesa_env,
 )
 from servo.util import host_triple
 
@@ -255,12 +255,9 @@ class MachCommands(CommandBase):
 
         packages.discard('stylo')
 
-        env = self.build_env()
+        env = self.build_env(test_unit=True)
         env["RUST_BACKTRACE"] = "1"
 
-        # Work around https://github.com/rust-lang/cargo/issues/4790
-        del env["RUSTDOCFLAGS"]
-
         if "msvc" in host_triple():
             # on MSVC, we need some DLLs in the path. They were copied
             # in to the servo.exe build dir, so just point PATH to that.
@@ -268,7 +265,7 @@ class MachCommands(CommandBase):
 
         features = self.servo_features()
         if len(packages) > 0:
-            args = ["cargo", "bench" if bench else "test"]
+            args = ["cargo", "bench" if bench else "test", "--manifest-path", self.servo_manifest()]
             for crate in packages:
                 args += ["-p", "%s_tests" % crate]
             for crate in in_crate_packages:
@@ -281,7 +278,7 @@ class MachCommands(CommandBase):
             if nocapture:
                 args += ["--", "--nocapture"]
 
-            err = call(args, env=env, cwd=self.servo_crate())
+            err = self.call_rustup_run(args, env=env)
             if err is not 0:
                 return err
 
@@ -293,17 +290,19 @@ class MachCommands(CommandBase):
     @CommandArgument('--release', default=False, action="store_true",
                      help="Run with a release build of servo")
     def test_stylo(self, release=False, test_name=None):
-        self.set_use_stable_rust()
+        self.set_use_geckolib_toolchain()
         self.ensure_bootstrapped()
 
         env = self.build_env()
         env["RUST_BACKTRACE"] = "1"
         env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8")
 
-        args = (["cargo", "test", "-p", "stylo_tests"] +
+        args = (
-                (["--release"] if release else []) + (test_name or []))
+            ["cargo", "test", "--manifest-path", self.geckolib_manifest(), "-p", "stylo_tests"] +
-        with cd(path.join("ports", "geckolib")):
+            (["--release"] if release else []) +
-            return call(args, env=env)
+            (test_name or [])
+        )
+        return self.call_rustup_run(args, env=env)
 
     @Command('test-content',
              description='Run the content tests',

python/servo/util.py

@@ -16,7 +16,6 @@ import shutil
 from socket import error as socket_error
 import StringIO
 import sys
-import tarfile
 import zipfile
 import urllib2
 import certifi
@@ -148,10 +147,8 @@ def download_file(desc, src, dst):
 
 
 def extract(src, dst, movedir=None):
-    if src.endswith(".zip"):
+    assert src.endswith(".zip")
-        zipfile.ZipFile(src).extractall(dst)
+    zipfile.ZipFile(src).extractall(dst)
-    else:
-        tarfile.open(src).extractall(dst)
 
     if movedir:
         for f in os.listdir(movedir):

servobuild.example

@@ -6,22 +6,11 @@
 
 # Tool options
 [tools]
-# Where Rust compiler and other downloads will be stored.  Can be
+# If use-rustup is set to false, mach will run for example "cargo build"
-# shared by multiple Servo repositories.  Defaults to <servo-repo>/.servo
+# instead of "rustup run --install <toolchain> cargo build"
-cache-dir = "./.servo"
+# It is then the user’s responsibility to ensure that cargo and especially rustc
-
+# in $PATH are versions compatible with Servo.
-# Where Cargo stores all of its clones  Defaults to <servo-repo>/.cargo
+use-rustup = true
-cargo-home-dir = "./.cargo"
-
-# If system-rust is true, will use rustc/rustdoc from the path, or if
-# rust-root is specified, will make sure that rust-root is in the path
-# when building. Similarly for Cargo. This takes care of PATH as well as
-# [DY]LD_LIBRARY_PATH.
-# rust-root and cargo-root default to <servo-repo>/
-system-rust = false
-#rust-root = "/path/to/rust"
-system-cargo = false
-#cargo-root = "/path/to/cargo"
 
 # If rustc-with-gold is true, will try to find and use gold linker with rustc.
 # Defaults to true