Make rustup a requirement and switch to rust-toolchain.toml (#30056)

This change makes rustup a requirement for building Servo with `./mach`
and switches to the newer `rust-toolchain.toml` format. The goal here is
to make mach builds more similar to non-mach builds.

- The new format allows listing the required components, removing some of
  the complexity from our mach scripts.
- This means we must raise the required version of rustup to 1.23. The
  current version is 1.26.
- We no longer wrap every call to cargo and rustc in "rustup run" calls
  as both cargo and rustc will take care of installing and using all
  necessary components specified in `rust-toolchain.toml` when run
  inside the project directory.

python/servo/bootstrap_commands.py

@@ -21,6 +21,8 @@ import sys
 import traceback
 import urllib
 
+import toml
+
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
@@ -283,15 +285,16 @@ class MachCommands(CommandBase):
                      default='1',
                      help='Keep up to this many most recent nightlies')
     def clean_nightlies(self, force=False, keep=None):
-        print("Current Rust version for Servo: {}".format(self.rust_toolchain()))
+        print(f"Current Rust version for Servo: {self.rust_toolchain()}")
         old_toolchains = []
         keep = int(keep)
-        stdout = subprocess.check_output(['git', 'log', '--format=%H', 'rust-toolchain'])
+        stdout = subprocess.check_output(['git', 'log', '--format=%H', 'rust-toolchain.toml'])
         for i, commit_hash in enumerate(stdout.split(), 1):
             if i > keep:
-                toolchain = subprocess.check_output(
+                toolchain_config_text = subprocess.check_output(
-                    ['git', 'show', '%s:rust-toolchain' % commit_hash])
+                    ['git', 'show', f'{commit_hash}:rust-toolchain.toml'])
-                old_toolchains.append(toolchain.strip())
+                toolchain = toml.loads(toolchain_config_text)['toolchain']['channel']
+                old_toolchains.append(toolchain)
 
         removing_anything = False
         stdout = subprocess.check_output(['rustup', 'toolchain', 'list'])
@@ -300,10 +303,10 @@ class MachCommands(CommandBase):
                 if toolchain_with_host.startswith(old):
                     removing_anything = True
                     if force:
-                        print("Removing {}".format(toolchain_with_host))
+                        print(f"Removing {toolchain_with_host}")
                         check_call(["rustup", "uninstall", toolchain_with_host])
                     else:
-                        print("Would remove {}".format(toolchain_with_host))
+                        print(f"Would remove {toolchain_with_host}")
         if not removing_anything:
             print("Nothing to remove.")
         elif not force:

python/servo/command_base.py

@@ -244,7 +244,6 @@ class CommandBase(object):
 
         context.sharedir = self.config["tools"]["cache-dir"]
 
-        self.config["tools"].setdefault("use-rustup", True)
         self.config["tools"].setdefault("rustc-with-gold", get_env_bool("SERVO_RUSTC_WITH_GOLD", True))
 
         self.config.setdefault("build", {})
@@ -272,24 +271,13 @@ class CommandBase(object):
     _rust_toolchain = None
 
     def rust_toolchain(self):
-        if self._rust_toolchain is None:
+        if self._rust_toolchain:
-            filename = path.join(self.context.topdir, "rust-toolchain")
+            return self._rust_toolchain
-            with open(filename) as f:
-                self._rust_toolchain = f.read().strip()
-
-            if platform.system() == "Windows":
-                self._rust_toolchain += "-x86_64-pc-windows-msvc"
 
+        toolchain_file = path.join(self.context.topdir, "rust-toolchain.toml")
+        self._rust_toolchain = toml.load(toolchain_file)['toolchain']['channel']
         return self._rust_toolchain
 
-    def call_rustup_run(self, args, **kwargs):
-        if self.config["tools"]["use-rustup"]:
-            assert self.context.bootstrapped
-            args = ["rustup" + BIN_SUFFIX, "run", "--install", self.rust_toolchain()] + args
-        else:
-            args[0] += BIN_SUFFIX
-        return call(args, **kwargs)
-
     def get_top_dir(self):
         return self.context.topdir
 
@@ -933,7 +921,7 @@ class CommandBase(object):
         if with_debug_assertions or self.config["build"]["debug-assertions"]:
             env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -C debug_assertions"
 
-        return self.call_rustup_run(["cargo", command] + args + cargo_args, env=env, verbose=verbose)
+        return call(["cargo", command] + args + cargo_args, env=env, verbose=verbose)
 
     def android_support_dir(self):
         return path.join(self.context.topdir, "support", "android")
@@ -986,43 +974,19 @@ class CommandBase(object):
             return True
         return False
 
-    def ensure_bootstrapped(self, rustup_components=None):
+    def ensure_bootstrapped(self):
         if self.context.bootstrapped:
             return
 
         servo.platform.get().passive_bootstrap()
 
-        if self.config["tools"]["use-rustup"]:
+        needs_toolchain_install = self.cross_compile_target \
-            self.ensure_rustup_version()
+            and self.cross_compile_target not in check_output(
-            toolchain = self.rust_toolchain()
+                ["rustup", "target", "list", "--installed"], cwd=self.context.topdir
-
-            status = subprocess.call(
-                ["rustup", "run", toolchain, "rustc", "--version"],
-                stdout=open(os.devnull, "wb"),
-                stderr=subprocess.STDOUT,
             )
-            if status:
+        if needs_toolchain_install:
-                check_call(["rustup", "toolchain", "install", "--profile", "minimal", toolchain])
+            check_call(["rustup", "target", "add", self.cross_compile_target],
-
+                       cwd=self.context.topdir)
-            installed = check_output(
-                ["rustup", "component", "list", "--installed", "--toolchain", toolchain]
-            )
-            required_components = {
-                # For components/script_plugins, https://github.com/rust-lang/rust/pull/67469
-                "rustc-dev",
-                # https://github.com/rust-lang/rust/issues/72594#issuecomment-633779564
-                "llvm-tools-preview",
-            }
-            for component in set(rustup_components or []) | required_components:
-                if component.encode("utf-8") not in installed:
-                    check_call(["rustup", "component", "add", "--toolchain", toolchain, component])
-
-            needs_toolchain_install = self.cross_compile_target \
-                and self.cross_compile_target.encode("utf-8") not in check_output(
-                    ["rustup", "target", "list", "--installed", "--toolchain", toolchain]
-                )
-            if needs_toolchain_install:
-                check_call(["rustup", "target", "add", "--toolchain", toolchain, self.cross_compile_target])
 
         self.context.bootstrapped = True
 
@@ -1042,7 +1006,7 @@ class CommandBase(object):
                 sys.exit(1)
             raise
         version = tuple(map(int, re.match(br"rustup (\d+)\.(\d+)\.(\d+)", version_line).groups()))
-        version_needed = (1, 21, 0)
+        version_needed = (1, 23, 0)
         if version < version_needed:
             print("rustup is at version %s.%s.%s, Servo requires %s.%s.%s or more recent." % (version + version_needed))
             print("Try running 'rustup self update'.")

python/servo/devenv_commands.py

@@ -97,7 +97,7 @@ class MachCommands(CommandBase):
 
         self.ensure_bootstrapped()
         with cd(self.context.topdir):
-            self.call_rustup_run(["cargo", "update"] + params, env=self.build_env())
+            call(["cargo", "update"] + params, env=self.build_env())
 
     @Command('rustc',
              description='Run the Rust compiler',
@@ -110,7 +110,7 @@ class MachCommands(CommandBase):
             params = []
 
         self.ensure_bootstrapped()
-        return self.call_rustup_run(["rustc"] + params, env=self.build_env())
+        return call(["rustc"] + params, env=self.build_env())
 
     @Command('cargo-fix',
              description='Run "cargo fix"',
@@ -173,10 +173,16 @@ class MachCommands(CommandBase):
     def rustup(self):
         nightly_date = urllib.request.urlopen(
             "https://static.rust-lang.org/dist/channel-rust-nightly-date.txt").read()
-        toolchain = b"nightly-" + nightly_date
+        new_toolchain = f"nightly-{nightly_date.decode('utf-8')}"
-        filename = path.join(self.context.topdir, "rust-toolchain")
+        old_toolchain = self.rust_toolchain()
-        with open(filename, "wb") as f:
+
-            f.write(toolchain + b"\n")
+        filename = path.join(self.context.topdir, "rust-toolchain.toml")
+        with open(filename, "r", encoding="utf-8") as file:
+            contents = file.read()
+        contents = contents.replace(old_toolchain, new_toolchain)
+        with open(filename, "w", encoding="utf-8") as file:
+            file.write(contents)
+
         self.ensure_bootstrapped()
 
     @Command('fetch',
@@ -184,9 +190,7 @@ class MachCommands(CommandBase):
              category='devenv')
     def fetch(self):
         self.ensure_bootstrapped()
-
+        return call(["cargo", "fetch"], env=self.build_env())
-        with cd(self.context.topdir):
-            return self.call_rustup_run(["cargo", "fetch"], env=self.build_env())
 
     @Command('ndk-stack',
              description='Invoke the ndk-stack tool with the expected symbol paths',

python/servo/post_build_commands.py

@@ -244,10 +244,8 @@ class PostBuildCommands(CommandBase):
         help="Command-line arguments to be passed through to cargo doc")
     @CommandBase.build_like_command_arguments
     def doc(self, params: List[str], **kwargs):
-        self.ensure_bootstrapped(rustup_components=["rust-docs"])
+        self.ensure_bootstrapped()
-        rustc_path = check_output(
+        rustc_path = check_output(["rustup" + BIN_SUFFIX, "which", "rustc"], cwd=self.context.topdir)
-            ["rustup" + BIN_SUFFIX, "which", "--toolchain", self.rust_toolchain(), "rustc"]
-        ).decode('utf-8')
         assert path.basename(path.dirname(rustc_path)) == "bin"
         toolchain_path = path.dirname(path.dirname(rustc_path))
         rust_docs = path.join(toolchain_path, "share", "doc", "rust", "html")

python/servo/testing_commands.py

@@ -263,14 +263,6 @@ class MachCommands(CommandBase):
               "tests/wpt/mozilla/.")
         return 0
 
-    def install_rustfmt(self):
-        self.ensure_bootstrapped()
-        with open(os.devnull, "w") as devnull:
-            if self.call_rustup_run(["cargo", "fmt", "--version", "-q"],
-                                    stderr=devnull) != 0:
-                # Rustfmt is not installed. Install:
-                self.call_rustup_run(["rustup", "component", "add", "rustfmt-preview"])
-
     @Command('test-tidy',
              description='Run the source code tidiness check',
              category='testing')
@@ -289,8 +281,7 @@ class MachCommands(CommandBase):
         else:
             manifest_dirty = wpt.manifestupdate.update(check_clean=True)
         tidy_failed = tidy.scan(not all_files, not no_progress, stylo=stylo, no_wpt=no_wpt)
-        self.install_rustfmt()
+        rustfmt_failed = call(["cargo", "fmt", "--", "--check"])
-        rustfmt_failed = self.call_rustup_run(["cargo", "fmt", "--", "--check"])
 
         if rustfmt_failed:
             print("Run `./mach fmt` to fix the formatting")
@@ -404,8 +395,7 @@ class MachCommands(CommandBase):
              description='Format the Rust and CPP source files with rustfmt',
              category='testing')
     def format_code(self):
-        self.install_rustfmt()
+        return call(["cargo", "fmt"])
-        return self.call_rustup_run(["cargo", "fmt"])
 
     @Command('update-wpt',
              description='Update the web platform tests',

rust-toolchain

@@ -1 +0,0 @@
-nightly-2023-02-01

rust-toolchain.toml

@@ -0,0 +1,11 @@
+[toolchain]
+channel = "nightly-2023-02-01"
+components = [
+    # https://github.com/rust-lang/rust/issues/72594#issuecomment-633779564
+    "llvm-tools-preview",
+    # For components/script_plugins, https://github.com/rust-lang/rust/pull/67469
+    "rustc-dev",
+    "rust-docs",
+    "rustfmt-preview",
+]
+profile = "minimal"

servobuild.example

@@ -4,12 +4,6 @@
 
 # Tool options
 [tools]
-# If use-rustup is set to false, mach will run for example "cargo build"
-# instead of "rustup run --install <toolchain> cargo build"
-# It is then the user’s responsibility to ensure that cargo and especially rustc
-# in $PATH are versions compatible with Servo.
-use-rustup = true
-
 # If rustc-with-gold is true, will try to find and use gold linker with rustc.
 # Defaults to true
 rustc-with-gold = true