Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-02 15:25:31 +00:00
Code Issues Projects Releases Packages Wiki Activity

Replace hsts preload list hashmap with an FST (#37015)

Browse source 
This reduces the memory used by the preload list to just 1.9MB. The
total memory savings in HSTS from
pre-103cbed928
is now 62MB, or 96%. And in terms of total resident memory is a 7.5%
reduction. The DAFSA/DAWG used by Firefox is 1.1MB so there could be
additional gains available but this seems like the best option based on
maintained libraries available (I could not find a good maintained
library for DAFSAs in Rust).

The main trick is this: the FST map API is currently designed to map
byte sequences to u64 values. Because we only need to determine if a
preloaded domain has the `includeSubdomains` flag set, we encode that
into the lowest bit of the ids in the map. This way finding an entry in
the map directly provides us with the `includeSubdomains` flag and we
don't need to keep another mapping in memory or on disk.

Updated the `./mach update-hsts-preload` command to generate the new FST
map file. (Not sure if I need to update any dev-dependencies anywhere
for this change)

This change also replaces the use of "mozilla.org" with "example.com" in
the HSTS unit tests to make sure that entries in the preload list do not
influence the tests (since example.com should not ever end up on the
preload list)

Testing: Updated unit tests
Fixes: #25929

---------

Signed-off-by: Sebastian C <sebsebmc@gmail.com>
This commit is contained in:
Sebastian C 2025-05-19 23:26:55 -05:00 • committed by GitHub
parent 3a6d3c7bed
commit 27c8a899ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 139 additions and 373010 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Cargo.lock generated
View file

@ -2325,6 +2325,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fst"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"
[[package]]
name = "futf"
version = "0.1.5"
@ -4809,6 +4815,7 @@ dependencies = [
"devtools_traits",
"embedder_traits",
"flate2",
"fst",
"futures 0.3.31",
"futures-core",
"futures-util",

2
components/net/Cargo.toml
View file

@ -29,6 +29,7 @@ crossbeam-channel = { workspace = true }
data-url = { workspace = true }
devtools_traits = { workspace = true }
embedder_traits = { workspace = true }
fst = "0.4"
futures = { version = "0.3", package = "futures" }
futures-core = { version = "0.3.30", default-features = false }
futures-util = { version = "0.3.30", default-features = false }
@ -77,6 +78,7 @@ webrender_api = { workspace = true }
[dev-dependencies]
embedder_traits = { workspace = true, features = ["baked-default-resources"] }
flate2 = "1"
fst = "0.4"
futures = { version = "0.3", features = ["compat"] }
hyper = { workspace = true, features = ["full"] }
hyper-util = { workspace = true, features = ["server-graceful"] }

106
components/net/hsts.rs
View file

@ -9,9 +9,11 @@ use std::sync::LazyLock;
use std::time::Duration;
use embedder_traits::resources::{self, Resource};
use fst::{Map, MapBuilder};
use headers::{HeaderMapExt, StrictTransportSecurity};
use http::HeaderMap;
use log::{debug, error, info};
use malloc_size_of::{MallocSizeOf, MallocSizeOfOps};
use malloc_size_of_derive::MallocSizeOf;
use net_traits::IncludeSubdomains;
use net_traits::pub_domains::reg_suffix;
@ -85,99 +87,67 @@ pub struct HstsList {
/// it is split out to allow sharing between the private and public http state
/// as well as potentially swpaping out the underlying type to something immutable
/// and more efficient like FSTs or DAFSA/DAWGs.
#[derive(Clone, Debug, Default, Deserialize, MallocSizeOf, Serialize)]
pub struct HstsPreloadList {
pub entries_map: HashMap<String, Vec<HstsEntry>>,
/// To generate a new version of the FST map file run `./mach update-hsts-preload`
#[derive(Clone, Debug)]
pub struct HstsPreloadList(pub fst::Map<Vec<u8>>);
impl MallocSizeOf for HstsPreloadList {
#[allow(unsafe_code)]
fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize {
unsafe { ops.malloc_size_of(self.0.as_fst().as_inner().as_ptr()) }
}
}
pub static PRELOAD_LIST_ENTRIES: LazyLock<HstsPreloadList> =
static PRELOAD_LIST_ENTRIES: LazyLock<HstsPreloadList> =
LazyLock::new(HstsPreloadList::from_servo_preload);
pub fn hsts_preload_size_of(ops: &mut MallocSizeOfOps) -> usize {
PRELOAD_LIST_ENTRIES.size_of(ops)
}
impl HstsPreloadList {
/// Create an `HstsList` from the bytes of a JSON preload file.
pub fn from_preload(preload_content: &str) -> Option<HstsPreloadList> {
#[derive(Deserialize)]
struct HstsEntries {
entries: Vec<HstsEntry>,
}
let hsts_entries: Option<HstsEntries> = serde_json::from_str(preload_content).ok();
hsts_entries.map(|hsts_entries| {
let mut hsts_list: HstsPreloadList = HstsPreloadList::default();
for hsts_entry in hsts_entries.entries {
hsts_list.push(hsts_entry);
}
hsts_list
})
pub fn from_preload(preload_content: Vec<u8>) -> Option<HstsPreloadList> {
Map::new(preload_content).map(HstsPreloadList).ok()
}
pub fn from_servo_preload() -> HstsPreloadList {
debug!("Intializing HSTS Preload list");
let list = resources::read_string(Resource::HstsPreloadList);
HstsPreloadList::from_preload(&list).unwrap_or_else(|| {
let map_bytes = resources::read_bytes(Resource::HstsPreloadList);
HstsPreloadList::from_preload(map_bytes).unwrap_or_else(|| {
error!("HSTS preload file is invalid. Setting HSTS list to default values");
HstsPreloadList {
entries_map: Default::default(),
}
HstsPreloadList(MapBuilder::memory().into_map())
})
}
pub fn is_host_secure(&self, host: &str) -> bool {
let base_domain = reg_suffix(host);
self.entries_map.get(base_domain).is_some_and(|entries| {
// No need to check for expiration in the preload list
entries.iter().any(|e| {
if e.include_subdomains {
e.matches_subdomain(host) || e.matches_domain(host)
} else {
e.matches_domain(host)
}
})
})
}
let parts = host[..host.len() - base_domain.len()].rsplit_terminator('.');
let mut domain_to_test = base_domain.to_owned();
pub fn has_domain(&self, host: &str, base_domain: &str) -> bool {
self.entries_map
.get(base_domain)
.is_some_and(|entries| entries.iter().any(|e| e.matches_domain(host)))
}
if self.0.get(&domain_to_test).is_some_and(|id| {
// The FST map ids were constructed such that the parity represents the includeSubdomain flag
id % 2 == 1 || domain_to_test == host
}) {
return true;
}
pub fn has_subdomain(&self, host: &str, base_domain: &str) -> bool {
self.entries_map.get(base_domain).is_some_and(|entries| {
entries
.iter()
.any(|e| e.include_subdomains && e.matches_subdomain(host))
})
}
pub fn push(&mut self, entry: HstsEntry) {
let host = entry.host.clone();
let base_domain = reg_suffix(&host);
let have_domain = self.has_domain(&entry.host, base_domain);
let have_subdomain = self.has_subdomain(&entry.host, base_domain);
let entries = self.entries_map.entry(base_domain.to_owned()).or_default();
if !have_domain && !have_subdomain {
entries.push(entry);
} else if !have_subdomain {
for e in entries {
if e.matches_domain(&entry.host) {
e.include_subdomains = entry.include_subdomains;
// TODO(sebsebmc): We could shrink the the HSTS preload memory use further by using a type
// that doesn't store an expiry since all preload entries should be "forever"
e.expires_at = entry.expires_at;
}
// Check all further subdomains up to the passed host
for part in parts {
domain_to_test = format!("{}.{}", part, domain_to_test);
if self.0.get(&domain_to_test).is_some_and(|id| {
// The FST map ids were constructed such that the parity represents the includeSubdomain flag
id % 2 == 1 || domain_to_test == host
}) {
return true;
}
}
false
}
}
impl HstsList {
pub fn is_host_secure(&self, host: &str) -> bool {
debug!("HSTS: is {host} secure?");
if PRELOAD_LIST_ENTRIES.is_host_secure(host) {
info!("{host} is in the preload list");
return true;

3
components/net/resource_thread.rs
View file

@ -21,7 +21,6 @@ use embedder_traits::EmbedderProxy;
use hyper_serde::Serde;
use ipc_channel::ipc::{self, IpcReceiver, IpcReceiverSet, IpcSender};
use log::{debug, trace, warn};
use malloc_size_of::MallocSizeOf;
use net_traits::blob_url_store::parse_blob_url;
use net_traits::filemanager_thread::FileTokenCheck;
use net_traits::pub_domains::public_suffix_list_size_of;
@ -292,7 +291,7 @@ impl ResourceChannelManager {
Report {
path: path!["hsts-preload-list"],
kind: ReportKind::ExplicitJemallocHeapSize,
size: hsts::PRELOAD_LIST_ENTRIES.size_of(ops),
size: hsts::hsts_preload_size_of(ops),
},
Report {
path: path!["public-suffix-list"],

159
components/net/tests/hsts.rs
View file

@ -6,13 +6,14 @@ use std::collections::HashMap;
use std::num::NonZeroU64;
use std::time::Duration as StdDuration;
use base64::Engine;
use net::hsts::{HstsEntry, HstsList, HstsPreloadList};
use net_traits::IncludeSubdomains;
#[test]
fn test_hsts_entry_is_not_expired_when_it_has_no_expires_at() {
let entry = HstsEntry {
host: "mozilla.org".to_owned(),
host: "example.com".to_owned(),
include_subdomains: false,
expires_at: None,
};
@ -23,7 +24,7 @@ fn test_hsts_entry_is_not_expired_when_it_has_no_expires_at() {
#[test]
fn test_hsts_entry_is_expired_when_it_has_reached_its_max_age() {
let entry = HstsEntry {
host: "mozilla.org".to_owned(),
host: "example.com".to_owned(),
include_subdomains: false,
expires_at: Some(NonZeroU64::new(1).unwrap()),
};
@ -59,7 +60,7 @@ fn test_base_domain_in_entries_map() {
list.push(
HstsEntry::new(
"servo.mozilla.org".to_owned(),
"servo.example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
@ -67,7 +68,7 @@ fn test_base_domain_in_entries_map() {
);
list.push(
HstsEntry::new(
"firefox.mozilla.org".to_owned(),
"firefox.example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
@ -75,7 +76,7 @@ fn test_base_domain_in_entries_map() {
);
list.push(
HstsEntry::new(
"bugzilla.org".to_owned(),
"example.org".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
@ -83,17 +84,17 @@ fn test_base_domain_in_entries_map() {
);
assert_eq!(list.entries_map.len(), 2);
assert_eq!(list.entries_map.get("mozilla.org").unwrap().len(), 2);
assert_eq!(list.entries_map.get("example.com").unwrap().len(), 2);
}
#[test]
fn test_push_entry_with_0_max_age_is_not_secure() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
"example.com".to_owned(),
vec![
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
Some(StdDuration::from_secs(500000)),
)
@ -106,23 +107,23 @@ fn test_push_entry_with_0_max_age_is_not_secure() {
list.push(
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
Some(StdDuration::ZERO),
)
.unwrap(),
);
assert_eq!(list.is_host_secure("mozilla.org"), false)
assert_eq!(list.is_host_secure("example.com"), false)
}
fn test_push_entry_with_0_max_age_evicts_entry_from_list() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
"example.com".to_owned(),
vec![
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
Some(StdDuration::from_secs(500000)),
)
@ -133,25 +134,25 @@ fn test_push_entry_with_0_max_age_evicts_entry_from_list() {
entries_map: entries_map,
};
assert_eq!(list.entries_map.get("mozilla.org").unwrap().len(), 1);
assert_eq!(list.entries_map.get("example.com").unwrap().len(), 1);
list.push(
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
Some(StdDuration::ZERO),
)
.unwrap(),
);
assert_eq!(list.entries_map.get("mozilla.org").unwrap().len(), 0);
assert_eq!(list.entries_map.get("example.com").unwrap().len(), 0);
}
#[test]
fn test_push_entry_to_hsts_list_should_not_add_subdomains_whose_superdomain_is_already_matched() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
vec![HstsEntry::new("mozilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap()],
"example.com".to_owned(),
vec![HstsEntry::new("example.com".to_owned(), IncludeSubdomains::Included, None).unwrap()],
);
let mut list = HstsList {
entries_map: entries_map,
@ -159,24 +160,24 @@ fn test_push_entry_to_hsts_list_should_not_add_subdomains_whose_superdomain_is_a
list.push(
HstsEntry::new(
"servo.mozilla.org".to_owned(),
"servo.example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
.unwrap(),
);
assert_eq!(list.entries_map.get("mozilla.org").unwrap().len(), 1)
assert_eq!(list.entries_map.get("example.com").unwrap().len(), 1)
}
#[test]
fn test_push_entry_to_hsts_list_should_add_subdomains_whose_superdomain_doesnt_include() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
"example.com".to_owned(),
vec![
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
@ -189,49 +190,49 @@ fn test_push_entry_to_hsts_list_should_add_subdomains_whose_superdomain_doesnt_i
list.push(
HstsEntry::new(
"servo.mozilla.org".to_owned(),
"servo.example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
.unwrap(),
);
assert_eq!(list.entries_map.get("mozilla.org").unwrap().len(), 2)
assert_eq!(list.entries_map.get("example.com").unwrap().len(), 2)
}
#[test]
fn test_push_entry_to_hsts_list_should_update_existing_domain_entrys_include_subdomains() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
vec![HstsEntry::new("mozilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap()],
"example.com".to_owned(),
vec![HstsEntry::new("example.com".to_owned(), IncludeSubdomains::Included, None).unwrap()],
);
let mut list = HstsList {
entries_map: entries_map,
};
assert!(list.is_host_secure("servo.mozilla.org"));
assert!(list.is_host_secure("servo.example.com"));
list.push(
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
.unwrap(),
);
assert!(!list.is_host_secure("servo.mozilla.org"))
assert!(!list.is_host_secure("servo.example.com"))
}
#[test]
fn test_push_entry_to_hsts_list_should_not_create_duplicate_entry() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
"example.com".to_owned(),
vec![
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
@ -244,14 +245,14 @@ fn test_push_entry_to_hsts_list_should_not_create_duplicate_entry() {
list.push(
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
.unwrap(),
);
assert_eq!(list.entries_map.get("mozilla.org").unwrap().len(), 1)
assert_eq!(list.entries_map.get("example.com").unwrap().len(), 1)
}
#[test]
@ -260,16 +261,14 @@ fn test_push_multiple_entrie_to_hsts_list_should_add_them_all() {
entries_map: HashMap::new(),
};
assert!(!list.is_host_secure("mozilla.org"));
assert!(!list.is_host_secure("bugzilla.org"));
assert!(!list.is_host_secure("example.com"));
assert!(!list.is_host_secure("example.org"));
list.push(HstsEntry::new("mozilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap());
list.push(
HstsEntry::new("bugzilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap(),
);
list.push(HstsEntry::new("example.com".to_owned(), IncludeSubdomains::Included, None).unwrap());
list.push(HstsEntry::new("example.org".to_owned(), IncludeSubdomains::Included, None).unwrap());
assert!(list.is_host_secure("mozilla.org"));
assert!(list.is_host_secure("bugzilla.org"));
assert!(list.is_host_secure("example.com"));
assert!(list.is_host_secure("example.org"));
}
#[test]
@ -278,25 +277,16 @@ fn test_push_entry_to_hsts_list_should_add_an_entry() {
entries_map: HashMap::new(),
};
assert!(!list.is_host_secure("mozilla.org"));
assert!(!list.is_host_secure("example.com"));
list.push(HstsEntry::new("mozilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap());
list.push(HstsEntry::new("example.com".to_owned(), IncludeSubdomains::Included, None).unwrap());
assert!(list.is_host_secure("mozilla.org"));
assert!(list.is_host_secure("example.com"));
}
#[test]
fn test_parse_hsts_preload_should_return_none_when_json_invalid() {
let mock_preload_content = "derp";
assert!(
HstsPreloadList::from_preload(mock_preload_content).is_none(),
"invalid preload list should not have parsed"
)
}
#[test]
fn test_parse_hsts_preload_should_return_none_when_json_contains_no_entries_map_key() {
let mock_preload_content = "{\"nothing\": \"to see here\"}";
let mock_preload_content = "derp".as_bytes().to_vec();
assert!(
HstsPreloadList::from_preload(mock_preload_content).is_none(),
"invalid preload list should not have parsed"
@ -305,20 +295,17 @@ fn test_parse_hsts_preload_should_return_none_when_json_contains_no_entries_map_
#[test]
fn test_parse_hsts_preload_should_decode_host_and_includes_subdomains() {
let mock_preload_content = "{\
\"entries\": [\
{\"host\": \"mozilla.org\",\
\"include_subdomains\": false}\
]\
}";
let hsts_list = HstsPreloadList::from_preload(mock_preload_content);
let entries_map = hsts_list.unwrap().entries_map;
// Generated with `fst map --sorted` on a csv of "example.com,0\nexample.org,3"
let mock_preload_content = base64::engine::general_purpose::STANDARD
.decode("AwAAAAAAAAAAAAAAAAAAAAAQkMQAEJfHAwABBW9jEQLNws/J0MXqwgIAAAAAAAAAJwAAAAAAAADVOFe6")
.unwrap();
let hsts_list = HstsPreloadList::from_preload(mock_preload_content).unwrap();
assert_eq!(
entries_map.get("mozilla.org").unwrap()[0].host,
"mozilla.org"
);
assert!(!entries_map.get("mozilla.org").unwrap()[0].include_subdomains);
assert_eq!(hsts_list.is_host_secure("derp"), false);
assert_eq!(hsts_list.is_host_secure("example.com"), true);
assert_eq!(hsts_list.is_host_secure("servo.example.com"), false);
assert_eq!(hsts_list.is_host_secure("example.org"), true);
assert_eq!(hsts_list.is_host_secure("servo.example.org"), true);
}
#[test]
@ -327,17 +314,17 @@ fn test_hsts_list_with_no_entries_map_does_not_is_host_secure() {
entries_map: HashMap::new(),
};
assert!(!hsts_list.is_host_secure("mozilla.org"));
assert!(!hsts_list.is_host_secure("example.com"));
}
#[test]
fn test_hsts_list_with_exact_domain_entry_is_is_host_secure() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
"example.com".to_owned(),
vec![
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
@ -349,31 +336,31 @@ fn test_hsts_list_with_exact_domain_entry_is_is_host_secure() {
entries_map: entries_map,
};
assert!(hsts_list.is_host_secure("mozilla.org"));
assert!(hsts_list.is_host_secure("example.com"));
}
#[test]
fn test_hsts_list_with_subdomain_when_include_subdomains_is_true_is_is_host_secure() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
vec![HstsEntry::new("mozilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap()],
"example.com".to_owned(),
vec![HstsEntry::new("example.com".to_owned(), IncludeSubdomains::Included, None).unwrap()],
);
let hsts_list = HstsList {
entries_map: entries_map,
};
assert!(hsts_list.is_host_secure("servo.mozilla.org"));
assert!(hsts_list.is_host_secure("servo.example.com"));
}
#[test]
fn test_hsts_list_with_subdomain_when_include_subdomains_is_false_is_not_is_host_secure() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
"example.com".to_owned(),
vec![
HstsEntry::new(
"mozilla.org".to_owned(),
"example.com".to_owned(),
IncludeSubdomains::NotIncluded,
None,
)
@ -384,44 +371,44 @@ fn test_hsts_list_with_subdomain_when_include_subdomains_is_false_is_not_is_host
entries_map: entries_map,
};
assert!(!hsts_list.is_host_secure("servo.mozilla.org"));
assert!(!hsts_list.is_host_secure("servo.example.com"));
}
#[test]
fn test_hsts_list_with_subdomain_when_host_is_not_a_subdomain_is_not_is_host_secure() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
vec![HstsEntry::new("mozilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap()],
"example.com".to_owned(),
vec![HstsEntry::new("example.com".to_owned(), IncludeSubdomains::Included, None).unwrap()],
);
let hsts_list = HstsList {
entries_map: entries_map,
};
assert!(!hsts_list.is_host_secure("servo-mozilla.org"));
assert!(!hsts_list.is_host_secure("servo-example.com"));
}
#[test]
fn test_hsts_list_with_subdomain_when_host_is_exact_match_is_is_host_secure() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
vec![HstsEntry::new("mozilla.org".to_owned(), IncludeSubdomains::Included, None).unwrap()],
"example.com".to_owned(),
vec![HstsEntry::new("example.com".to_owned(), IncludeSubdomains::Included, None).unwrap()],
);
let hsts_list = HstsList {
entries_map: entries_map,
};
assert!(hsts_list.is_host_secure("mozilla.org"));
assert!(hsts_list.is_host_secure("example.com"));
}
#[test]
fn test_hsts_list_with_expired_entry_is_not_is_host_secure() {
let mut entries_map = HashMap::new();
entries_map.insert(
"mozilla.org".to_owned(),
"example.com".to_owned(),
vec![HstsEntry {
host: "mozilla.org".to_owned(),
host: "example.com".to_owned(),
include_subdomains: false,
expires_at: Some(NonZeroU64::new(1).unwrap()),
}],
@ -430,11 +417,11 @@ fn test_hsts_list_with_expired_entry_is_not_is_host_secure() {
entries_map: entries_map,
};
assert!(!hsts_list.is_host_secure("mozilla.org"));
assert!(!hsts_list.is_host_secure("example.com"));
}
#[test]
fn test_preload_hsts_domains_well_formed() {
let hsts_list = HstsPreloadList::from_servo_preload();
assert!(!hsts_list.entries_map.is_empty());
assert_ne!(hsts_list.0.len(), 0);
}

4
components/shared/embedder/resources.rs
View file

@ -116,7 +116,7 @@ impl Resource {
match self {
Resource::BluetoothBlocklist => "gatt_blocklist.txt",
Resource::DomainList => "public_domains.txt",
Resource::HstsPreloadList => "hsts_preload.json",
Resource::HstsPreloadList => "hsts_preload.fstmap",
Resource::BadCertHTML => "badcert.html",
Resource::NetErrorHTML => "neterror.html",
Resource::RippyPNG => "rippy.png",
@ -155,7 +155,7 @@ fn resources_for_tests() -> Box<dyn ResourceReaderMethods + Sync + Send> {
&include_bytes!("../../../resources/public_domains.txt")[..]
},
Resource::HstsPreloadList => {
&include_bytes!("../../../resources/hsts_preload.json")[..]
&include_bytes!("../../../resources/hsts_preload.fstmap")[..]
},
Resource::BadCertHTML => &include_bytes!("../../../resources/badcert.html")[..],
Resource::NetErrorHTML => &include_bytes!("../../../resources/neterror.html")[..],

2
ports/servoshell/egl/android/resources.rs
View file

@ -17,7 +17,7 @@ impl ResourceReaderMethods for ResourceReaderInstance {
fn read(&self, res: Resource) -> Vec<u8> {
Vec::from(match res {
Resource::HstsPreloadList => {
&include_bytes!("../../../../resources/hsts_preload.json")[..]
&include_bytes!("../../../../resources/hsts_preload.fstmap")[..]
},
Resource::BadCertHTML => &include_bytes!("../../../../resources/badcert.html")[..],
Resource::NetErrorHTML => &include_bytes!("../../../../resources/neterror.html")[..],

32
python/servo/bootstrap_commands.py
View file

@ -8,6 +8,7 @@
# except according to those terms.
import base64
import csv
import glob
import json
import os
@ -15,6 +16,7 @@ import os.path as path
import re
import subprocess
import sys
import tempfile
import traceback
import urllib
@ -75,11 +77,11 @@ class MachCommands(CommandBase):
description='Download the HSTS preload list',
category='bootstrap')
def bootstrap_hsts_preload(self, force=False):
preload_filename = "hsts_preload.json"
preload_filename = "hsts_preload.fstmap"
preload_path = path.join(self.context.topdir, "resources")
chromium_hsts_url = "https://chromium.googlesource.com/chromium/src" + \
"/net/+/master/http/transport_security_state_static.json?format=TEXT"
"/+/main/net/http/transport_security_state_static.json?format=TEXT"
try:
content_base64 = download_bytes("Chromium HSTS preload list", chromium_hsts_url)
@ -87,28 +89,22 @@ class MachCommands(CommandBase):
print("Unable to download chromium HSTS preload list; are you connected to the internet?")
sys.exit(1)
content_decoded = base64.b64decode(content_base64)
content_decoded = base64.b64decode(content_base64).decode()
# The chromium "json" has single line comments in it which, of course,
# are non-standard/non-valid json. Simply strip them out before parsing
content_json = re.sub(r'(^|\s+)//.*$', '', content_decoded, flags=re.MULTILINE)
try:
pins_and_static_preloads = json.loads(content_json)
entries = {
"entries": [
{
"host": e["name"],
"include_subdomains": e.get("include_subdomains", False)
}
for e in pins_and_static_preloads["entries"]
]
}
with open(path.join(preload_path, preload_filename), 'w') as fd:
json.dump(entries, fd, indent=4)
except ValueError:
print("Unable to parse chromium HSTS preload list, has the format changed?")
with tempfile.NamedTemporaryFile(mode="w") as csv_file:
writer = csv.writer(csv_file)
for idx, e in enumerate(sorted(pins_and_static_preloads["entries"], key=lambda e: e["name"])):
next_id = idx * 2 + (1 if e.get("include_subdomains", False) else 0)
writer.writerow([e["name"], str(next_id)])
csv_file.flush()
subprocess.run(["fst", "map", "--sorted", csv_file.name, path.join(preload_path, preload_filename)])
except ValueError as e:
print(f"Unable to parse chromium HSTS preload list, has the format changed? \n{e}")
sys.exit(1)
@Command('update-pub-domains',

BIN
resources/hsts_preload.fstmap Normal file
View file

Binary file not shown.

372832
resources/hsts_preload.json
View file

File diff suppressed because it is too large Load diff

2
servo-tidy.toml
View file

@ -7,7 +7,7 @@ check-alphabetical-order = true
# Files that are ignored for all tidy and lint checks.
files = [
"./components/net/tests/parsable_mime/text",
"./resources/hsts_preload.json",
"./resources/hsts_preload.fstmap",
"./tests/wpt/meta/MANIFEST.json",
"./tests/wpt/mozilla/meta/MANIFEST.json",
# Long encoded string