Fix crown on NixOS (#30987)

* Fix crown on NixOS

* no need to install libs, because there are none

* fix program name in usage message

* use the cargo provided by each buildPhase

* cargo update --offline can be used to reformat lockfiles

* document how to keep rust-toolchain.toml and etc/shell.nix in sync

* clarify comment about allowBuiltinFetchGit

* fix license

* clarify purpose of filterlock

* explain why crown must not use workspace dependencies

docs/HACKING_QUICKSTART.md

@@ -174,6 +174,8 @@ about build scripts like
   "Could not run \`PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=\\"1\\" PKG_CONFIG_ALLOW_SYSTEM_LIBS=\\"1\\"
   \\"pkg-config\\" \\"--libs\\" \\"--cflags\\" \\"fontconfig\\"\`
 
+* (if you are on NixOS) [ERROR rust_analyzer::main_loop] FetchWorkspaceError: rust-analyzer failed to load workspace: Failed to load the project at /path/to/servo/Cargo.toml: Failed to read Cargo metadata from Cargo.toml file /path/to/servo/Cargo.toml, Some(Version { major: 1, minor: 74, patch: 1 }): Failed to run `cd "/path/to/servo" && "cargo" "metadata" "--format-version" "1" "--manifest-path" "/path/to/servo/Cargo.toml" "--filter-platform" "x86_64-unknown-linux-gnu"`: `cargo metadata` exited with an error: error: could not execute process `crown -vV` (never executed)
+
 This is because the rustflags (flags passed to the rust compiler) that standard `cargo` provides are
 different to what `./mach` uses, and so every time Servo is built using `cargo` it will undo all the
 work done by `./mach` (and vice versa).
@@ -203,8 +205,18 @@ the amount of disc space used).
 }
 ```
 
-If you are on NixOS, these settings should be enough to not need to run `code .` from within a
-`nix-shell etc/shell.nix`, but it wouldn’t hurt to try that if you still have problems.
+If you are on NixOS, you should also set CARGO_BUILD_RUSTC in `.vscode/settings.json` as follows,
+where `/nix/store/.../crown` is the output of `nix-shell etc/shell.nix --run 'command -v crown'`.
+These settings should be enough to not need to run `code .` from within a `nix-shell etc/shell.nix`,
+but it wouldn’t hurt to try that if you still have problems.
+
+```
+{
+    "rust-analyzer.server.extraEnv": {
+        "CARGO_BUILD_RUSTC": "/nix/store/.../crown",
+    },
+}
+```
 
 When enabling rust-analyzer’s proc macro support, you may start to see errors like
 

etc/shell.nix

@@ -1,12 +1,23 @@
 # This provides a shell with all the necesarry packages required to run mach and build servo
 # NOTE: This does not work offline or for nix-build
 
-with import <nixpkgs> {};
+with import <nixpkgs> {
+  overlays = [
+    (import (builtins.fetchTarball {
+      # Bumped the channel in rust-toolchain.toml? Bump this commit too!
+      url = "https://github.com/oxalica/rust-overlay/archive/a0df72e106322b67e9c6e591fe870380bd0da0d5.tar.gz";
+    }))
+  ];
+};
 let
-    pinnedSha = "6adf48f53d819a7b6e15672817fa1e78e5f4e84f";
     pinnedNixpkgs = import (builtins.fetchTarball {
-        url = "https://github.com/NixOS/nixpkgs/archive/${pinnedSha}.tar.gz";
+      url = "https://github.com/NixOS/nixpkgs/archive/6adf48f53d819a7b6e15672817fa1e78e5f4e84f.tar.gz";
     }) {};
+    rustToolchain = rust-bin.fromRustupToolchainFile ../rust-toolchain.toml;
+    rustPlatform = makeRustPlatform {
+      cargo = rustToolchain;
+      rustc = rustToolchain;
+    };
 in
 clangStdenv.mkDerivation rec {
   name = "servo-env";
@@ -33,6 +44,62 @@ clangStdenv.mkDerivation rec {
     # slow as it behaves as if -j1 was passed.
     # See https://github.com/servo/mozjs/issues/375
     pinnedNixpkgs.gnumake
+
+    # crown needs to be in our Cargo workspace so we can test it with `mach test`. This means its
+    # dependency tree is listed in the main Cargo.lock, making it awkward to build with Nix because
+    # all of Servo’s dependencies get pulled into the Nix store too, wasting over 1GB of disk space.
+    # Filtering the lockfile to only the parts needed by crown saves space and builds faster.
+    (let
+      vendorTarball = rustPlatform.fetchCargoTarball {
+        src = ../support/filterlock;
+        hash = "sha256-/kJNDtmv2uI7Qlmpi3DMWSw88rzEJSbroO0/QrgQrSc=";
+      };
+      vendorConfig = builtins.toFile "toml" ''
+        [source.crates-io]
+        replace-with = "vendor"
+        [source.vendor]
+        directory = "vendor"
+      '';
+
+      # Build and run filterlock over the main Cargo.lock.
+      filteredLockFile = (clangStdenv.mkDerivation {
+        name = "lock";
+        buildInputs = [ rustToolchain ];
+        src = ../support/filterlock;
+        buildPhase = ''
+          tar xzf ${vendorTarball}
+          mv cargo-deps-vendor.tar.gz vendor
+          mkdir .cargo
+          cp -- ${vendorConfig} .cargo/config.toml
+          > $out cargo run --offline -- ${../Cargo.lock} crown
+        '';
+      });
+    in (rustPlatform.buildRustPackage rec {
+      name = "crown";
+      src = ../support/crown;
+      doCheck = false;
+      cargoLock = {
+        lockFileContents = builtins.readFile filteredLockFile;
+
+        # Needed when not filtering (filteredLockFile = ../Cargo.lock), else we’ll get errors like
+        # “error: No hash was found while vendoring the git dependency blurmac-0.1.0.”
+        # allowBuiltinFetchGit = true;
+      };
+
+      # Copy the filtered lockfile, making it writable by cargo --offline.
+      postPatch = ''
+        install -m 644 ${filteredLockFile} Cargo.lock
+      '';
+
+      # Reformat the filtered lockfile, so that cargo --frozen won’t complain
+      # about the lockfile being dirty.
+      # TODO maybe this can be avoided by using toml_edit in filterlock?
+      preConfigure = ''
+        cargo update --offline
+      '';
+
+      RUSTC_BOOTSTRAP = "crown";
+    }))
   ] ++ (lib.optionals stdenv.isDarwin [
     darwin.apple_sdk.frameworks.AppKit
   ]);

rust-toolchain.toml

@@ -1,5 +1,7 @@
 [toolchain]
+# Be sure to update etc/shell.nix when bumping this!
 channel = "1.74"
+
 components = [
     # For support/crown
     "llvm-tools",

support/crown/Cargo.toml

@@ -5,6 +5,9 @@ version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
 
+# Do not use workspace dependencies in this package!
+# In etc/shell.nix, we filter Cargo.lock and build this package in isolation,
+# so it needs to make sense without the workspace manifest.
 [dev-dependencies]
 compiletest_rs = { version = "0.10", features = ["tmp"] }
 once_cell = "1"

support/filterlock/Cargo.lock

@@ -0,0 +1,147 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "filterlock"
+version = "0.1.0"
+dependencies = [
+ "serde",
+ "toml",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
+
+[[package]]
+name = "indexmap"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "memchr"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.194"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.194"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_spanned"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "toml"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "winnow"
+version = "0.5.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c"
+dependencies = [
+ "memchr",
+]

support/filterlock/Cargo.toml

@@ -0,0 +1,12 @@
+[workspace]
+
+[package]
+name = "filterlock"
+authors = ["The Servo Project Developers"]
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]
+serde = { version = "1.0.194", features = ["derive"] }
+toml = { version = "0.8.8", features = ["preserve_order"] }

support/filterlock/src/main.rs

@@ -0,0 +1,80 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+//! Filter the given lockfile to only the given package and its dependencies.
+//!
+//! Usage: `filterlock <path/to/Cargo.lock> <package>`
+//!
+//! This helper is used only by the Nix shell environment (etc/shell.nix).
+
+use std::{fs::File, env::{args_os, args}, io::Read, collections::BTreeSet};
+
+use serde::{Deserialize, Serialize};
+use toml::{map::Map, Value};
+
+#[derive(Deserialize, Serialize)]
+struct LockFile {
+    package: Vec<Package>,
+    #[serde(flatten)]
+    other: Map<String, Value>,
+}
+
+#[derive(Deserialize, Serialize)]
+struct Package {
+    name: String,
+    version: String,
+    dependencies: Option<Vec<String>>,
+    #[serde(flatten)]
+    other: Map<String, Value>,
+}
+
+fn main() {
+    let usage = "Usage: filterlock <path/to/Cargo.lock> <package>";
+    let path = args_os().nth(1).expect(usage);
+    let package = args().nth(2).expect(usage);
+
+    let mut file = File::open(path).expect("Failed to open lockfile");
+    let mut toml = String::new();
+    file.read_to_string(&mut toml).expect("Failed to read lockfile");
+    let toml: LockFile = toml::from_str(&toml).expect("Failed to parse lockfile");
+
+    // Find the closure of the given package and its dependencies.
+    let mut keep = BTreeSet::new();
+    let mut queue = vec![
+        toml.package.iter()
+            .find(|p| p.matches(&package))
+            .expect("Failed to find package"),
+    ];
+    while !queue.is_empty() {
+        let package = queue.pop().expect("Guaranteed by while");
+        keep.insert((package.name.clone(), package.version.clone()));
+        if let Some(dependencies) = package.dependencies.as_ref() {
+            for dependency in dependencies {
+                let package = toml.package.iter()
+                    .find(|p| p.matches(&dependency))
+                    .expect("Failed to find package");
+                queue.push(package);
+            }
+        }
+    }
+
+    // Remove packages that are not in the closure.
+    let mut toml = toml;
+    let filtered_packages = toml.package.drain(..)
+        .filter(|p| keep.contains(&(p.name.clone(), p.version.clone())))
+        .collect();
+    let toml = LockFile { package: filtered_packages, ..toml };
+
+    println!("{}", toml::to_string(&toml).expect("Failed to serialise lockfile"));
+}
+
+impl Package {
+    fn matches(&self, spec: &str) -> bool {
+        if let Some((name, version)) = spec.split_once(" ") {
+            self.name == name && self.version == version
+        } else {
+            self.name == spec
+        }
+    }
+}