mirror of
https://github.com/servo/servo.git
synced 2025-07-25 00:00:20 +01:00
732ae5714f
Urg! The Chromium HSTS preload JSON file contains single line comments. Previously these were filtered out with a very simple regex that just looked for '//' and removed the line. Now the file has added a couple fields that have URLs in them that were erroneously removed and caused the JSON parsing to fail. This commit slightly complicates the regex to fix this specific problem. If this happens again, it's likely worth it to figure out how to use a real parser to remove the comments. servo/servo#8760
331 lines
13 KiB
Python
331 lines
13 KiB
Python
# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
|
|
# file at the top-level directory of this distribution.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
# option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
from __future__ import print_function, unicode_literals
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import os.path as path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import StringIO
|
|
import tarfile
|
|
import urllib2
|
|
from distutils.version import LooseVersion
|
|
|
|
from mach.decorators import (
|
|
CommandArgument,
|
|
CommandProvider,
|
|
Command,
|
|
)
|
|
|
|
from servo.command_base import CommandBase, cd, host_triple
|
|
|
|
|
|
def download(desc, src, writer):
|
|
print("Downloading %s..." % desc)
|
|
dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
|
|
|
|
try:
|
|
resp = urllib2.urlopen(src)
|
|
|
|
fsize = None
|
|
if resp.info().getheader('Content-Length'):
|
|
fsize = int(resp.info().getheader('Content-Length').strip())
|
|
|
|
recved = 0
|
|
chunk_size = 8192
|
|
|
|
while True:
|
|
chunk = resp.read(chunk_size)
|
|
if not chunk:
|
|
break
|
|
recved += len(chunk)
|
|
if not dumb:
|
|
if fsize is not None:
|
|
pct = recved * 100.0 / fsize
|
|
print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
|
|
|
|
sys.stdout.flush()
|
|
writer.write(chunk)
|
|
|
|
if not dumb:
|
|
print()
|
|
except urllib2.HTTPError, e:
|
|
print("Download failed (%d): %s - %s" % (e.code, e.reason, src))
|
|
|
|
cpu_type = subprocess.check_output(["uname", "-m"]).strip().lower()
|
|
if e.code == 404 and cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
|
|
# i686
|
|
print("Note: Servo does not currently bootstrap 32bit snapshots of Rust")
|
|
print("See https://github.com/servo/servo/issues/3899")
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
def download_file(desc, src, dst):
|
|
with open(dst, 'wb') as fd:
|
|
download(desc, src, fd)
|
|
|
|
|
|
def download_bytes(desc, src):
|
|
content_writer = StringIO.StringIO()
|
|
download(desc, src, content_writer)
|
|
return content_writer.getvalue()
|
|
|
|
|
|
def extract(src, dst, movedir=None):
|
|
tarfile.open(src).extractall(dst)
|
|
|
|
if movedir:
|
|
for f in os.listdir(movedir):
|
|
frm = path.join(movedir, f)
|
|
to = path.join(dst, f)
|
|
os.rename(frm, to)
|
|
os.rmdir(movedir)
|
|
|
|
os.remove(src)
|
|
|
|
|
|
@CommandProvider
|
|
class MachCommands(CommandBase):
|
|
@Command('env',
|
|
description='Print environment setup commands',
|
|
category='bootstrap')
|
|
def env(self):
|
|
env = self.build_env()
|
|
print("export PATH=%s" % env["PATH"])
|
|
if sys.platform == "darwin":
|
|
print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
|
|
else:
|
|
print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])
|
|
|
|
@Command('bootstrap-rust',
|
|
description='Download the Rust compiler snapshot',
|
|
category='bootstrap')
|
|
@CommandArgument('--force', '-f',
|
|
action='store_true',
|
|
help='Force download even if a snapshot already exists')
|
|
def bootstrap_rustc(self, force=False):
|
|
rust_dir = path.join(
|
|
self.context.sharedir, "rust", self.rust_snapshot_path())
|
|
if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc")):
|
|
print("Snapshot Rust compiler already downloaded.", end=" ")
|
|
print("Use |bootstrap-rust --force| to download again.")
|
|
return
|
|
|
|
if path.isdir(rust_dir):
|
|
shutil.rmtree(rust_dir)
|
|
os.makedirs(rust_dir)
|
|
|
|
date = self.rust_snapshot_path().split("/")[0]
|
|
install_dir = path.join(self.context.sharedir, "rust", date)
|
|
|
|
# The Rust compiler is hosted on the nightly server under the date with a name
|
|
# rustc-nightly-HOST-TRIPLE.tar.gz. We just need to pull down and extract it,
|
|
# giving a directory name that will be the same as the tarball name (rustc is
|
|
# in that directory).
|
|
snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s.tar.gz"
|
|
% self.rust_snapshot_path())
|
|
tgz_file = rust_dir + '-rustc.tar.gz'
|
|
|
|
download_file("Rust compiler", snapshot_url, tgz_file)
|
|
|
|
print("Extracting Rust compiler...")
|
|
extract(tgz_file, install_dir)
|
|
|
|
# Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz`, with
|
|
# a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
|
|
# This `lib` directory needs to be extracted and merged with the `rustc/lib`
|
|
# directory from the host compiler above.
|
|
# TODO: make it possible to request an additional cross-target to add to this
|
|
# list.
|
|
stdlibs = [host_triple(), "arm-linux-androideabi"]
|
|
for target in stdlibs:
|
|
snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s/rust-std-nightly-%s.tar.gz"
|
|
% (date, target))
|
|
tgz_file = install_dir + ('rust-std-nightly-%s.tar.gz' % target)
|
|
|
|
download_file("Host rust library for target %s" % target, snapshot_url, tgz_file)
|
|
print("Extracting Rust stdlib for target %s..." % target)
|
|
extract(tgz_file, install_dir)
|
|
shutil.copytree(path.join(install_dir, "rust-std-nightly-%s" % target,
|
|
"rust-std-%s" % target, "lib", "rustlib", target),
|
|
path.join(install_dir, "rustc-nightly-%s" % host_triple(),
|
|
"rustc", "lib", "rustlib", target))
|
|
shutil.rmtree(path.join(install_dir, "rust-std-nightly-%s" % target))
|
|
|
|
print("Snapshot Rust ready.")
|
|
|
|
@Command('bootstrap-rust-docs',
|
|
description='Download the Rust documentation',
|
|
category='bootstrap')
|
|
@CommandArgument('--force', '-f',
|
|
action='store_true',
|
|
help='Force download even if docs already exist')
|
|
def bootstrap_rustc_docs(self, force=False):
|
|
self.ensure_bootstrapped()
|
|
rust_root = self.config["tools"]["rust-root"]
|
|
docs_dir = path.join(rust_root, "doc")
|
|
if not force and path.exists(docs_dir):
|
|
print("Snapshot Rust docs already downloaded.", end=" ")
|
|
print("Use |bootstrap-rust-docs --force| to download again.")
|
|
return
|
|
|
|
if path.isdir(docs_dir):
|
|
shutil.rmtree(docs_dir)
|
|
docs_name = self.rust_snapshot_path().replace("rustc-", "rust-docs-")
|
|
snapshot_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-docs-nightly-%s.tar.gz"
|
|
% host_triple())
|
|
tgz_file = path.join(rust_root, 'doc.tar.gz')
|
|
|
|
download_file("Rust docs", snapshot_url, tgz_file)
|
|
|
|
print("Extracting Rust docs...")
|
|
temp_dir = path.join(rust_root, "temp_docs")
|
|
if path.isdir(temp_dir):
|
|
shutil.rmtree(temp_dir)
|
|
extract(tgz_file, temp_dir)
|
|
shutil.move(path.join(temp_dir, docs_name.split("/")[1],
|
|
"rust-docs", "share", "doc", "rust", "html"),
|
|
docs_dir)
|
|
shutil.rmtree(temp_dir)
|
|
print("Rust docs ready.")
|
|
|
|
@Command('bootstrap-cargo',
|
|
description='Download the Cargo build tool',
|
|
category='bootstrap')
|
|
@CommandArgument('--force', '-f',
|
|
action='store_true',
|
|
help='Force download even if cargo already exists')
|
|
def bootstrap_cargo(self, force=False):
|
|
cargo_dir = path.join(self.context.sharedir, "cargo",
|
|
self.cargo_build_id())
|
|
if not force and path.exists(path.join(cargo_dir, "bin", "cargo")):
|
|
print("Cargo already downloaded.", end=" ")
|
|
print("Use |bootstrap-cargo --force| to download again.")
|
|
return
|
|
|
|
if path.isdir(cargo_dir):
|
|
shutil.rmtree(cargo_dir)
|
|
os.makedirs(cargo_dir)
|
|
|
|
tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
|
|
nightly_url = "https://static-rust-lang-org.s3.amazonaws.com/cargo-dist/%s/%s" % \
|
|
(self.cargo_build_id(), tgz_file)
|
|
|
|
download_file("Cargo nightly", nightly_url, tgz_file)
|
|
|
|
print("Extracting Cargo nightly...")
|
|
nightly_dir = path.join(cargo_dir,
|
|
path.basename(tgz_file).replace(".tar.gz", ""))
|
|
extract(tgz_file, cargo_dir, movedir=nightly_dir)
|
|
print("Cargo ready.")
|
|
|
|
@Command('update-hsts-preload',
|
|
description='Download the HSTS preload list',
|
|
category='bootstrap')
|
|
def bootstrap_hsts_preload(self, force=False):
|
|
preload_filename = "hsts_preload.json"
|
|
preload_path = path.join(self.context.topdir, "resources")
|
|
|
|
chromium_hsts_url = "https://chromium.googlesource.com/chromium/src" + \
|
|
"/net/+/master/http/transport_security_state_static.json?format=TEXT"
|
|
|
|
try:
|
|
content_base64 = download_bytes("Chromium HSTS preload list", chromium_hsts_url)
|
|
except urllib2.URLError:
|
|
print("Unable to download chromium HSTS preload list; are you connected to the internet?")
|
|
sys.exit(1)
|
|
|
|
content_decoded = base64.b64decode(content_base64)
|
|
|
|
# The chromium "json" has single line comments in it which, of course,
|
|
# are non-standard/non-valid json. Simply strip them out before parsing
|
|
content_json = re.sub(r'(^|\s+)//.*$', '', content_decoded, flags=re.MULTILINE)
|
|
|
|
try:
|
|
pins_and_static_preloads = json.loads(content_json)
|
|
entries = {
|
|
"entries": [
|
|
{
|
|
"host": e["name"],
|
|
"include_subdomains": e.get("include_subdomains", False)
|
|
}
|
|
for e in pins_and_static_preloads["entries"]
|
|
]
|
|
}
|
|
|
|
with open(path.join(preload_path, preload_filename), 'w') as fd:
|
|
json.dump(entries, fd, indent=4)
|
|
except ValueError, e:
|
|
print("Unable to parse chromium HSTS preload list, has the format changed?")
|
|
sys.exit(1)
|
|
|
|
@Command('update-submodules',
|
|
description='Update submodules',
|
|
category='bootstrap')
|
|
def update_submodules(self):
|
|
# Ensure that the installed git version is >= 1.8.1
|
|
gitversion_output = subprocess.check_output(["git", "--version"])
|
|
gitversion = LooseVersion(gitversion_output.split(" ")[-1])
|
|
if gitversion < LooseVersion("1.8.1"):
|
|
print("Git version 1.8.1 or above required. Current version is {}"
|
|
.format(gitversion))
|
|
sys.exit(1)
|
|
submodules = subprocess.check_output(["git", "submodule", "status"])
|
|
for line in submodules.split('\n'):
|
|
components = line.strip().split(' ')
|
|
if len(components) > 1:
|
|
module_path = components[1]
|
|
if path.exists(module_path):
|
|
with cd(module_path):
|
|
output = subprocess.check_output(
|
|
["git", "status", "--porcelain"])
|
|
if len(output) != 0:
|
|
print("error: submodule %s is not clean"
|
|
% module_path)
|
|
print("\nClean the submodule and try again.")
|
|
return 1
|
|
subprocess.check_call(
|
|
["git", "submodule", "--quiet", "sync", "--recursive"])
|
|
subprocess.check_call(
|
|
["git", "submodule", "update", "--init", "--recursive"])
|
|
|
|
@Command('clean-snapshots',
|
|
description='Clean unused snapshots of Rust and Cargo',
|
|
category='bootstrap')
|
|
@CommandArgument('--force', '-f',
|
|
action='store_true',
|
|
help='Actually remove stuff')
|
|
def clean_snapshots(self, force=False):
|
|
rust_current = self.rust_snapshot_path().split('/')[0]
|
|
cargo_current = self.cargo_build_id()
|
|
print("Current Rust version: " + rust_current)
|
|
print("Current Cargo version: " + cargo_current)
|
|
removing_anything = False
|
|
for current, base in [(rust_current, "rust"), (cargo_current, "cargo")]:
|
|
base = path.join(self.context.sharedir, base)
|
|
for name in os.listdir(base):
|
|
if name != current:
|
|
removing_anything = True
|
|
name = path.join(base, name)
|
|
if force:
|
|
print("Removing " + name)
|
|
shutil.rmtree(name)
|
|
else:
|
|
print("Would remove " + name)
|
|
if not removing_anything:
|
|
print("Nothing to remove.")
|
|
elif not force:
|
|
print("Nothing done. "
|
|
"Run `./mach clean-snapshots -f` to actually remove.")
|