Auto merge of #25417 - servo:profile, r=jdm,Manishearth

mach: automatically use the 'minimal' rustup profile + rustc-dev component

Fixes https://github.com/servo/servo/issues/25401
Fixes https://github.com/servo/servo/issues/25402

etc/taskcluster/decision_task.py

@@ -181,9 +181,6 @@ def linux_tidy_unit_untrusted():
         .with_dockerfile(dockerfile_path("build"))
         .with_env(**build_env, **unix_build_env, **linux_build_env)
         .with_repo_bundle()
-        .with_script("rustup set profile minimal")
-        # required by components/script_plugins:
-        .with_script("rustup component add rustc-dev")
         .with_script("""
             ./mach test-tidy --no-progress --all
             ./mach test-tidy --no-progress --self-test
@@ -229,7 +226,6 @@ def linux_docs_check():
         linux_build_task("Docs + check")
         .with_treeherder("Linux x64", "Doc+Check")
         .with_script("""
-            rustup component add rust-docs
             RUSTDOCFLAGS="--disable-minification" ./mach doc
             (
                 cd target/doc
@@ -304,11 +300,10 @@ def with_rust_nightly():
         modified_build_env["RUSTFLAGS"] = " ".join(flags)
 
     return (
-        linux_build_task("with Rust Nightly", build_env=modified_build_env, install_rustc_dev=False)
+        linux_build_task("with Rust Nightly", build_env=modified_build_env)
         .with_treeherder("Linux x64", "RustNightly")
         .with_script("""
             echo "nightly" > rust-toolchain
-            rustup component add rustc-dev
             ./mach build --dev
             ./mach test-unit
         """)
@@ -850,7 +845,7 @@ def macos_task(name):
     )
 
 
-def linux_build_task(name, *, build_env=build_env, install_rustc_dev=True):
+def linux_build_task(name, *, build_env=build_env):
     task = (
         linux_task(name)
         # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
@@ -867,14 +862,8 @@ def linux_build_task(name, *, build_env=build_env, install_rustc_dev=True):
         .with_dockerfile(dockerfile_path("build"))
         .with_env(**build_env, **unix_build_env, **linux_build_env)
         .with_repo_bundle()
-        .with_script("""
-            rustup set profile minimal
-            ./mach bootstrap-gstreamer
-        """)
+        .with_script("./mach bootstrap-gstreamer")
     )
-    if install_rustc_dev:
-        # required by components/script_plugins:
-        task = task.with_script("rustup component add rustc-dev")
     return task
 
 
@@ -884,9 +873,9 @@ def android_build_task(name):
         # file: NDK parses $(file $SHELL) to tell x64 host from x86
         # wget: servo-media-gstreamer’s build script
         .with_script("""
-            apt-get update -q
-            apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget
-            ./mach bootstrap-android --accept-all-licences --build
+            time apt-get update -q
+            time apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget
+            time ./mach bootstrap-android --accept-all-licences --build
         """)
     )
 
@@ -920,9 +909,6 @@ def windows_build_task(name, package=True, arch="x86_64"):
             path="python3",
         )
         .with_rustup()
-        .with_script("rustup set profile minimal")
-        # required by components/script_plugins:
-        .with_script("rustup component add rustc-dev")
     )
     if arch in hashes["non-devel"] and arch in hashes["devel"]:
         task = (
@@ -968,11 +954,6 @@ def macos_build_task(name):
         .with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects")
         .with_python2()
         .with_rustup()
-        # Since macOS workers are long-lived and ~/.rustup kept across tasks:
-        .with_script("rustup self update")
-        .with_script("rustup set profile minimal")
-        # required by components/script_plugins:
-        .with_script("rustup component add rustc-dev")
         .with_index_and_artifacts_expire_in(build_artifacts_expire_in)
         # Debugging for surprising generic-worker behaviour
         .with_early_script("ls")

etc/taskcluster/decisionlib.py

@@ -693,6 +693,7 @@ class MacOsGenericWorkerTask(UnixTaskMixin, GenericWorkerTask):
         return self.with_early_script("""
             export PATH="$HOME/.cargo/bin:$PATH"
             which rustup || curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y
+            rustup self update
         """)
 
 

etc/taskcluster/docker/build.dockerfile

@@ -44,7 +44,8 @@ RUN \
         #
     && \
     #
-    #
+    # Install the version of rustup that is current when this Docker image is being built:
+    # We want at least 1.21 (increment in this comment to force an image rebuild).
     curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none --profile=minimal -y && \
     #
     #

python/servo/bootstrap_commands.py

@@ -281,8 +281,7 @@ class MachCommands(CommandBase):
                      default='1',
                      help='Keep up to this many most recent nightlies')
     def clean_nightlies(self, force=False, keep=None):
-        default_toolchain = self.default_toolchain()
-        print("Current Rust version for Servo: {}".format(default_toolchain))
+        print("Current Rust version for Servo: {}".format(self.rust_toolchain()))
         old_toolchains = []
         keep = int(keep)
         stdout = subprocess.check_output(['git', 'log', '--format=%H', 'rust-toolchain'])

python/servo/build_commands.py

@@ -30,7 +30,7 @@ from mach.decorators import (
 from mach.registrar import Registrar
 
 from mach_bootstrap import _get_exec_path
-from servo.command_base import CommandBase, cd, call, check_call, BIN_SUFFIX, append_to_path_env, gstreamer_root
+from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root
 from servo.util import host_triple
 
 
@@ -237,14 +237,6 @@ class MachCommands(CommandBase):
         if very_verbose:
             opts += ["-vv"]
 
-        if target:
-            if self.config["tools"]["use-rustup"] and not uwp:
-                # 'rustup target add' fails if the toolchain is not installed at all.
-                self.call_rustup_run(["rustc", "--version"])
-
-                check_call(["rustup" + BIN_SUFFIX, "target", "add",
-                            "--toolchain", self.toolchain(), target])
-
         env = self.build_env(target=target, is_build=True, uwp=uwp, features=features)
         self.ensure_bootstrapped(target=target)
         self.ensure_clobbered()

python/servo/command_base.py

@@ -345,38 +345,23 @@ class CommandBase(object):
         # Set default android target
         self.handle_android_target("armv7-linux-androideabi")
 
-    _default_toolchain = None
+    _rust_toolchain = None
 
-    def toolchain(self):
-        return self.default_toolchain()
-
-    def default_toolchain(self):
-        if self._default_toolchain is None:
+    def rust_toolchain(self):
+        if self._rust_toolchain is None:
             filename = path.join(self.context.topdir, "rust-toolchain")
             with open(filename) as f:
-                self._default_toolchain = f.read().strip()
-        return self._default_toolchain
+                self._rust_toolchain = f.read().strip()
+
+            if platform.system() == "Windows":
+                self._rust_toolchain += "-x86_64-pc-windows-msvc"
+
+        return self._rust_toolchain
 
     def call_rustup_run(self, args, **kwargs):
         if self.config["tools"]["use-rustup"]:
-            try:
-                version_line = subprocess.check_output(["rustup" + BIN_SUFFIX, "--version"])
-            except OSError as e:
-                if e.errno == NO_SUCH_FILE_OR_DIRECTORY:
-                    print("It looks like rustup is not installed. See instructions at "
-                          "https://github.com/servo/servo/#setting-up-your-environment")
-                    print()
-                    return 1
-                raise
-            version = tuple(map(int, re.match(b"rustup (\d+)\.(\d+)\.(\d+)", version_line).groups()))
-            if version < (1, 11, 0):
-                print("rustup is at version %s.%s.%s, Servo requires 1.11.0 or more recent." % version)
-                print("Try running 'rustup self update'.")
-                return 1
-            toolchain = self.toolchain()
-            if platform.system() == "Windows":
-                toolchain += "-x86_64-pc-windows-msvc"
-            args = ["rustup" + BIN_SUFFIX, "run", "--install", toolchain] + args
+            assert self.context.bootstrapped
+            args = ["rustup" + BIN_SUFFIX, "run", "--install", self.rust_toolchain()] + args
         else:
             args[0] += BIN_SUFFIX
         return call(args, **kwargs)
@@ -1009,7 +994,7 @@ install them, let us know by filing a bug!")
             return True
         return False
 
-    def ensure_bootstrapped(self, target=None):
+    def ensure_bootstrapped(self, target=None, rustup_components=None):
         if self.context.bootstrapped:
             return
 
@@ -1019,8 +1004,43 @@ install them, let us know by filing a bug!")
         if "msvc" in target_platform:
             Registrar.dispatch("bootstrap", context=self.context)
 
+        if self.config["tools"]["use-rustup"]:
+            self.ensure_rustup_version()
+            toolchain = self.rust_toolchain()
+
+            if toolchain.encode("utf-8") not in check_output(["rustup", "toolchain", "list"]):
+                check_call(["rustup", "toolchain", "install", "--profile", "minimal", toolchain])
+
+            installed = check_output(
+                ["rustup", "component", "list", "--installed", "--toolchain", toolchain]
+            )
+            for component in set(rustup_components or []) | {"rustc-dev"}:
+                if component.encode("utf-8") not in installed:
+                    check_call(["rustup", "component", "add", "--toolchain", toolchain, component])
+
+            if target and "uwp" not in target and target not in check_output(
+                ["rustup", "target", "list", "--installed", "--toolchain", toolchain]
+            ):
+                check_call(["rustup", "target", "add", "--toolchain", toolchain, target])
+
         self.context.bootstrapped = True
 
+    def ensure_rustup_version(self):
+        try:
+            version_line = subprocess.check_output(["rustup" + BIN_SUFFIX, "--version"])
+        except OSError as e:
+            if e.errno == NO_SUCH_FILE_OR_DIRECTORY:
+                print("It looks like rustup is not installed. See instructions at "
+                      "https://github.com/servo/servo/#setting-up-your-environment")
+                print()
+                return 1
+            raise
+        version = tuple(map(int, re.match(b"rustup (\d+)\.(\d+)\.(\d+)", version_line).groups()))
+        if version < (1, 21, 0):
+            print("rustup is at version %s.%s.%s, Servo requires 1.11.0 or more recent." % version)
+            print("Try running 'rustup self update'.")
+            return 1
+
     def ensure_clobbered(self, target_dir=None):
         if target_dir is None:
             target_dir = self.get_target_dir()

python/servo/devenv_commands.py

@@ -24,7 +24,7 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd, call, BIN_SUFFIX
+from servo.command_base import CommandBase, cd, call
 from servo.build_commands import notify_build_done
 from servo.util import get_static_rust_lang_org_dist, get_urlopen_kwargs
 
@@ -49,7 +49,7 @@ class MachCommands(CommandBase):
 
         features += self.pick_media_stack(media_stack, target)
 
-        self.ensure_bootstrapped()
+        self.ensure_bootstrapped(target=target)
         self.ensure_clobbered()
         env = self.build_env()
 
@@ -213,7 +213,7 @@ class MachCommands(CommandBase):
         filename = path.join(self.context.topdir, "rust-toolchain")
         with open(filename, "w") as f:
             f.write(toolchain + "\n")
-        return call(["rustup" + BIN_SUFFIX, "component", "add", "rustc-dev"])
+        self.ensure_bootstrapped()
 
     @Command('fetch',
              description='Fetch Rust, Cargo and Cargo dependencies',

python/servo/post_build_commands.py

@@ -241,14 +241,13 @@ class PostBuildCommands(CommandBase):
     @CommandBase.build_like_command_arguments
     def doc(self, params, features, target=None, android=False, magicleap=False,
             media_stack=None, **kwargs):
-        env = os.environ.copy()
-        env["RUSTUP_TOOLCHAIN"] = self.toolchain()
-        rustc_path = check_output(["rustup" + BIN_SUFFIX, "which", "rustc"], env=env)
+        self.ensure_bootstrapped(rustup_components=["rust-docs"])
+        rustc_path = check_output(
+            ["rustup" + BIN_SUFFIX, "which", "--toolchain", self.rust_toolchain(), "rustc"])
         assert path.basename(path.dirname(rustc_path)) == "bin"
         toolchain_path = path.dirname(path.dirname(rustc_path))
         rust_docs = path.join(toolchain_path, "share", "doc", "rust", "html")
 
-        self.ensure_bootstrapped()
         docs = path.join(self.get_target_dir(), "doc")
         if not path.exists(docs):
             os.makedirs(docs)

python/servo/testing_commands.py

@@ -199,7 +199,6 @@ class MachCommands(CommandBase):
     def test_perf(self, base=None, date=None, submit=False):
         self.set_software_rendering_env(True, False)
 
-        self.ensure_bootstrapped()
         env = self.build_env()
         cmd = ["bash", "test_perf.sh"]
         if base:
@@ -298,7 +297,7 @@ class MachCommands(CommandBase):
                 args += ["--", "--nocapture"]
 
             err = self.run_cargo_build_like_command("bench" if bench else "test", args, env=env, **kwargs)
-            if err is not 0:
+            if err:
                 return err
 
     @Command('test-content',
@@ -310,6 +309,7 @@ class MachCommands(CommandBase):
         return 0
 
     def install_rustfmt(self):
+        self.ensure_bootstrapped()
         with open(os.devnull, "w") as devnull:
             if self.call_rustup_run(["cargo", "fmt", "--version", "-q"],
                                     stderr=devnull) != 0:
@@ -369,8 +369,6 @@ class MachCommands(CommandBase):
     @CommandArgument('tests', default=None, nargs="...",
                      help="Specific tests to run, relative to the tests directory")
     def test_webidl(self, quiet, tests):
-        self.ensure_bootstrapped()
-
         test_file_dir = path.abspath(path.join(PROJECT_TOPLEVEL_PATH, "components", "script",
                                                "dom", "bindings", "codegen", "parser"))
         # For the `import WebIDL` in runtests.py
@@ -388,7 +386,6 @@ class MachCommands(CommandBase):
              category='testing',
              parser=create_parser_wpt)
     def test_wpt_failure(self, **kwargs):
-        self.ensure_bootstrapped()
         kwargs["pause_after_test"] = False
         kwargs["include"] = ["infrastructure/failing-test.html"]
         return not self._test_wpt(**kwargs)
@@ -398,7 +395,6 @@ class MachCommands(CommandBase):
              category='testing',
              parser=create_parser_wpt)
     def test_wpt(self, **kwargs):
-        self.ensure_bootstrapped()
         ret = self.run_test_list_or_dispatch(kwargs["test_list"], "wpt", self._test_wpt, **kwargs)
         if kwargs["always_succeed"]:
             return 0
@@ -498,7 +494,6 @@ class MachCommands(CommandBase):
              category='testing',
              parser=updatecommandline.create_parser())
     def update_wpt(self, **kwargs):
-        self.ensure_bootstrapped()
         run_file = path.abspath(path.join("tests", "wpt", "update.py"))
         patch = kwargs.get("patch", False)
 
@@ -716,7 +711,6 @@ class MachCommands(CommandBase):
                           str(c1).ljust(width_col3), str(d1).ljust(width_col4)))
 
     def jquery_test_runner(self, cmd, release, dev):
-        self.ensure_bootstrapped()
         base_dir = path.abspath(path.join("tests", "jquery"))
         jquery_dir = path.join(base_dir, "jquery")
         run_file = path.join(base_dir, "run_jquery.py")
@@ -736,7 +730,6 @@ class MachCommands(CommandBase):
         return call([run_file, cmd, bin_path, base_dir])
 
     def dromaeo_test_runner(self, tests, release, dev):
-        self.ensure_bootstrapped()
         base_dir = path.abspath(path.join("tests", "dromaeo"))
         dromaeo_dir = path.join(base_dir, "dromaeo")
         run_file = path.join(base_dir, "run_dromaeo.py")
@@ -969,8 +962,6 @@ testing/web-platform/mozilla/tests for Servo-only tests""" % reference_path)
     @CommandArgument('--version', default='2.0.0',
                      help='WebGL conformance suite version')
     def update_webgl(self, version=None):
-        self.ensure_bootstrapped()
-
         base_dir = path.abspath(path.join(PROJECT_TOPLEVEL_PATH,
                                 "tests", "wpt", "mozilla", "tests", "webgl"))
         run_file = path.join(base_dir, "tools", "import-conformance-tests.py")