diff --git a/docs/HACKING_QUICKSTART.md b/docs/HACKING_QUICKSTART.md index 65dbd5ca6af..d9e9c065909 100644 --- a/docs/HACKING_QUICKSTART.md +++ b/docs/HACKING_QUICKSTART.md @@ -174,6 +174,8 @@ about build scripts like "Could not run \`PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=\\"1\\" PKG_CONFIG_ALLOW_SYSTEM_LIBS=\\"1\\" \\"pkg-config\\" \\"--libs\\" \\"--cflags\\" \\"fontconfig\\"\` +* (if you are on NixOS) [ERROR rust_analyzer::main_loop] FetchWorkspaceError: rust-analyzer failed to load workspace: Failed to load the project at /path/to/servo/Cargo.toml: Failed to read Cargo metadata from Cargo.toml file /path/to/servo/Cargo.toml, Some(Version { major: 1, minor: 74, patch: 1 }): Failed to run `cd "/path/to/servo" && "cargo" "metadata" "--format-version" "1" "--manifest-path" "/path/to/servo/Cargo.toml" "--filter-platform" "x86_64-unknown-linux-gnu"`: `cargo metadata` exited with an error: error: could not execute process `crown -vV` (never executed) + This is because the rustflags (flags passed to the rust compiler) that standard `cargo` provides are different to what `./mach` uses, and so every time Servo is built using `cargo` it will undo all the work done by `./mach` (and vice versa). @@ -203,8 +205,18 @@ the amount of disc space used). } ``` -If you are on NixOS, these settings should be enough to not need to run `code .` from within a -`nix-shell etc/shell.nix`, but it wouldn’t hurt to try that if you still have problems. +If you are on NixOS, you should also set CARGO_BUILD_RUSTC in `.vscode/settings.json` as follows, +where `/nix/store/.../crown` is the output of `nix-shell etc/shell.nix --run 'command -v crown'`. +These settings should be enough to not need to run `code .` from within a `nix-shell etc/shell.nix`, +but it wouldn’t hurt to try that if you still have problems. + +``` +{ + "rust-analyzer.server.extraEnv": { + "CARGO_BUILD_RUSTC": "/nix/store/.../crown", + }, +} +``` When enabling rust-analyzer’s proc macro support, you may start to see errors like diff --git a/etc/shell.nix b/etc/shell.nix index 743dc5271c5..0eebfcf2692 100644 --- a/etc/shell.nix +++ b/etc/shell.nix @@ -1,12 +1,23 @@ # This provides a shell with all the necesarry packages required to run mach and build servo # NOTE: This does not work offline or for nix-build -with import {}; +with import { + overlays = [ + (import (builtins.fetchTarball { + # Bumped the channel in rust-toolchain.toml? Bump this commit too! + url = "https://github.com/oxalica/rust-overlay/archive/a0df72e106322b67e9c6e591fe870380bd0da0d5.tar.gz"; + })) + ]; +}; let - pinnedSha = "6adf48f53d819a7b6e15672817fa1e78e5f4e84f"; pinnedNixpkgs = import (builtins.fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/${pinnedSha}.tar.gz"; + url = "https://github.com/NixOS/nixpkgs/archive/6adf48f53d819a7b6e15672817fa1e78e5f4e84f.tar.gz"; }) {}; + rustToolchain = rust-bin.fromRustupToolchainFile ../rust-toolchain.toml; + rustPlatform = makeRustPlatform { + cargo = rustToolchain; + rustc = rustToolchain; + }; in clangStdenv.mkDerivation rec { name = "servo-env"; @@ -33,6 +44,62 @@ clangStdenv.mkDerivation rec { # slow as it behaves as if -j1 was passed. # See https://github.com/servo/mozjs/issues/375 pinnedNixpkgs.gnumake + + # crown needs to be in our Cargo workspace so we can test it with `mach test`. This means its + # dependency tree is listed in the main Cargo.lock, making it awkward to build with Nix because + # all of Servo’s dependencies get pulled into the Nix store too, wasting over 1GB of disk space. + # Filtering the lockfile to only the parts needed by crown saves space and builds faster. + (let + vendorTarball = rustPlatform.fetchCargoTarball { + src = ../support/filterlock; + hash = "sha256-/kJNDtmv2uI7Qlmpi3DMWSw88rzEJSbroO0/QrgQrSc="; + }; + vendorConfig = builtins.toFile "toml" '' + [source.crates-io] + replace-with = "vendor" + [source.vendor] + directory = "vendor" + ''; + + # Build and run filterlock over the main Cargo.lock. + filteredLockFile = (clangStdenv.mkDerivation { + name = "lock"; + buildInputs = [ rustToolchain ]; + src = ../support/filterlock; + buildPhase = '' + tar xzf ${vendorTarball} + mv cargo-deps-vendor.tar.gz vendor + mkdir .cargo + cp -- ${vendorConfig} .cargo/config.toml + > $out cargo run --offline -- ${../Cargo.lock} crown + ''; + }); + in (rustPlatform.buildRustPackage rec { + name = "crown"; + src = ../support/crown; + doCheck = false; + cargoLock = { + lockFileContents = builtins.readFile filteredLockFile; + + # Needed when not filtering (filteredLockFile = ../Cargo.lock), else we’ll get errors like + # “error: No hash was found while vendoring the git dependency blurmac-0.1.0.” + # allowBuiltinFetchGit = true; + }; + + # Copy the filtered lockfile, making it writable by cargo --offline. + postPatch = '' + install -m 644 ${filteredLockFile} Cargo.lock + ''; + + # Reformat the filtered lockfile, so that cargo --frozen won’t complain + # about the lockfile being dirty. + # TODO maybe this can be avoided by using toml_edit in filterlock? + preConfigure = '' + cargo update --offline + ''; + + RUSTC_BOOTSTRAP = "crown"; + })) ] ++ (lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.AppKit ]); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 9c2fd1a734f..8807a1bccd9 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,7 @@ [toolchain] +# Be sure to update etc/shell.nix when bumping this! channel = "1.74" + components = [ # For support/crown "llvm-tools", diff --git a/support/crown/Cargo.toml b/support/crown/Cargo.toml index 19a60cc59e8..9010fb3f10c 100644 --- a/support/crown/Cargo.toml +++ b/support/crown/Cargo.toml @@ -5,6 +5,9 @@ version = "0.1.0" edition = "2021" license = "MPL-2.0" +# Do not use workspace dependencies in this package! +# In etc/shell.nix, we filter Cargo.lock and build this package in isolation, +# so it needs to make sense without the workspace manifest. [dev-dependencies] compiletest_rs = { version = "0.10", features = ["tmp"] } once_cell = "1" diff --git a/support/filterlock/Cargo.lock b/support/filterlock/Cargo.lock new file mode 100644 index 00000000000..629920f6209 --- /dev/null +++ b/support/filterlock/Cargo.lock @@ -0,0 +1,147 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "filterlock" +version = "0.1.0" +dependencies = [ + "serde", + "toml", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "proc-macro2" +version = "1.0.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "2.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "winnow" +version = "0.5.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c" +dependencies = [ + "memchr", +] diff --git a/support/filterlock/Cargo.toml b/support/filterlock/Cargo.toml new file mode 100644 index 00000000000..048110ed063 --- /dev/null +++ b/support/filterlock/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] + +[package] +name = "filterlock" +authors = ["The Servo Project Developers"] +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +serde = { version = "1.0.194", features = ["derive"] } +toml = { version = "0.8.8", features = ["preserve_order"] } diff --git a/support/filterlock/src/main.rs b/support/filterlock/src/main.rs new file mode 100644 index 00000000000..5788987f0c1 --- /dev/null +++ b/support/filterlock/src/main.rs @@ -0,0 +1,80 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +//! Filter the given lockfile to only the given package and its dependencies. +//! +//! Usage: `filterlock ` +//! +//! This helper is used only by the Nix shell environment (etc/shell.nix). + +use std::{fs::File, env::{args_os, args}, io::Read, collections::BTreeSet}; + +use serde::{Deserialize, Serialize}; +use toml::{map::Map, Value}; + +#[derive(Deserialize, Serialize)] +struct LockFile { + package: Vec, + #[serde(flatten)] + other: Map, +} + +#[derive(Deserialize, Serialize)] +struct Package { + name: String, + version: String, + dependencies: Option>, + #[serde(flatten)] + other: Map, +} + +fn main() { + let usage = "Usage: filterlock "; + let path = args_os().nth(1).expect(usage); + let package = args().nth(2).expect(usage); + + let mut file = File::open(path).expect("Failed to open lockfile"); + let mut toml = String::new(); + file.read_to_string(&mut toml).expect("Failed to read lockfile"); + let toml: LockFile = toml::from_str(&toml).expect("Failed to parse lockfile"); + + // Find the closure of the given package and its dependencies. + let mut keep = BTreeSet::new(); + let mut queue = vec![ + toml.package.iter() + .find(|p| p.matches(&package)) + .expect("Failed to find package"), + ]; + while !queue.is_empty() { + let package = queue.pop().expect("Guaranteed by while"); + keep.insert((package.name.clone(), package.version.clone())); + if let Some(dependencies) = package.dependencies.as_ref() { + for dependency in dependencies { + let package = toml.package.iter() + .find(|p| p.matches(&dependency)) + .expect("Failed to find package"); + queue.push(package); + } + } + } + + // Remove packages that are not in the closure. + let mut toml = toml; + let filtered_packages = toml.package.drain(..) + .filter(|p| keep.contains(&(p.name.clone(), p.version.clone()))) + .collect(); + let toml = LockFile { package: filtered_packages, ..toml }; + + println!("{}", toml::to_string(&toml).expect("Failed to serialise lockfile")); +} + +impl Package { + fn matches(&self, spec: &str) -> bool { + if let Some((name, version)) = spec.split_once(" ") { + self.name == name && self.version == version + } else { + self.name == spec + } + } +}